{
  "$schema": "https://raw.githubusercontent.com/VyomKulshrestha/Ferrum-OS/main/schemas/benchmarks.schema.json",
  "benchmark_scope": {
    "comparability": "Each section has a distinct protocol; cross-section ranking is invalid.",
    "latest_tagged_software_release": "v0.1.1",
    "source_channel": "current main evidence plus frozen research artifacts"
  },
  "canonical_url": "https://github.com/VyomKulshrestha/Ferrum-OS/blob/main/benchmarks.json",
  "current_ring3_preview": {
    "accelerator": "whpx",
    "evidence_grade": "repeated-emulator-measurement",
    "heap_growth_bytes_each_run": [
      0,
      0,
      0
    ],
    "horizons": [
      {
        "blocked_previews_each_run": [
          2,
          2,
          2
        ],
        "horizon": 1,
        "mean_microseconds_range": [
          1290,
          1400
        ],
        "median_microseconds_across_runs": [
          1000,
          1000,
          1000
        ],
        "median_run_mean_microseconds": 1320.0,
        "p95_microseconds_across_runs": [
          2000,
          2000,
          2000
        ]
      },
      {
        "blocked_previews_each_run": [
          2,
          2,
          2
        ],
        "horizon": 2,
        "mean_microseconds_range": [
          1350,
          1490
        ],
        "median_microseconds_across_runs": [
          1000,
          1000,
          1000
        ],
        "median_run_mean_microseconds": 1390.0,
        "p95_microseconds_across_runs": [
          2000,
          2000,
          2000
        ]
      },
      {
        "blocked_previews_each_run": [
          2,
          2,
          2
        ],
        "horizon": 3,
        "mean_microseconds_range": [
          1370,
          1450
        ],
        "median_microseconds_across_runs": [
          1000,
          1000,
          1000
        ],
        "median_run_mean_microseconds": 1430.0,
        "p95_microseconds_across_runs": [
          2000,
          2000,
          2000
        ]
      },
      {
        "blocked_previews_each_run": [
          2,
          2,
          2
        ],
        "horizon": 4,
        "mean_microseconds_range": [
          1400,
          1470
        ],
        "median_microseconds_across_runs": [
          1000,
          1000,
          1000
        ],
        "median_run_mean_microseconds": 1430.0,
        "p95_microseconds_across_runs": [
          2000,
          2000,
          2000
        ]
      },
      {
        "blocked_previews_each_run": [
          2,
          2,
          2
        ],
        "horizon": 5,
        "mean_microseconds_range": [
          1430,
          1570
        ],
        "median_microseconds_across_runs": [
          1000,
          1000,
          2000
        ],
        "median_run_mean_microseconds": 1510.0,
        "p95_microseconds_across_runs": [
          2000,
          2000,
          2000
        ]
      }
    ],
    "iterations_per_horizon_per_run": 100,
    "model_load_milliseconds_range": [
      26.0,
      30.0
    ],
    "protocol_id": "ring3-preview-whpx-2026-08-13",
    "runs": 3,
    "scope": "ring-3 Heliox capture + encoder + transition + safety predicate preview; no action dispatch",
    "source_commit": "54e1cc1819d3b3c11436f437733d197d469d1607",
    "warmup_previews_per_run": 64
  },
  "cyber_physical_software": {
    "claim_boundary": "Local deterministic software regression on the named commit. It does not prove an installed Gazebo or Webots deployment, real ROS 2/MQTT/CAN infrastructure, physical clocks, actuator hardware, robot safety, live EEG, hard-real-time behavior, native hypervisor containment, certification, or independent replication.",
    "contract_suites": [
      {
        "command": "cargo test --manifest-path userland/physical-runtime/Cargo.toml --target x86_64-pc-windows-msvc",
        "component": "ferrum-physical-runtime",
        "exit_code": 0,
        "failed": 0,
        "passed": 117
      },
      {
        "command": "cargo test --manifest-path userland/neural-protocol/Cargo.toml --target x86_64-pc-windows-msvc",
        "component": "ferrum-neural-protocol",
        "exit_code": 0,
        "failed": 0,
        "passed": 9
      },
      {
        "command": "python -m unittest discover -s tools/neurod -p test_*.py",
        "component": "neurod",
        "exit_code": 0,
        "failed": 0,
        "passed": 15
      },
      {
        "command": "python -m unittest tools.physical_sim_bridge.test_bridge",
        "component": "physical-simulator-bridge",
        "exit_code": 0,
        "failed": 0,
        "passed": 11
      }
    ],
    "contract_tests_failed": 0,
    "contract_tests_passed": 152,
    "covered_software_boundaries": [
      "versioned provenance and logical clocks",
      "deterministic sessions replay and fault injection",
      "virtual sensor actuator EEG watchdog and stop contracts",
      "scripted Gazebo ROS 2 and Webots bridge boundaries",
      "deterministic supervisor watchdog recovery and stop priority",
      "ROS 2 MQTT and CAN transport conformance",
      "actuator-disabled delivery",
      "bounded neural intent and proposal-only physical goals",
      "shadow-only physical JEPA evaluation",
      "host-managed agent-cell contract",
      "privacy retention reliability and incident primitives"
    ],
    "environment": {
      "cargo": "cargo 1.96.1 (356927216 2026-06-26)",
      "host_os": "Microsoft Windows NT 10.0.26220.0",
      "python_runtime": "CPython 3.12.6",
      "rust_target": "x86_64-pc-windows-msvc",
      "rustc": "rustc 1.96.1 (31fca3adb 2026-06-26)"
    },
    "evidence_grade": "local-deterministic-regression",
    "model_and_decoder_gates": [
      {
        "command": "python scripts/verify_physical_world_model.py --quick",
        "component": "physical-world-model",
        "exit_code": 0,
        "failed": 0,
        "passed": 21
      },
      {
        "command": "python scripts/verify_physical_jepa_robustness.py",
        "component": "physical-jepa-robustness",
        "exit_code": 0,
        "failed": 0,
        "passed": 8
      },
      {
        "command": "python scripts/evaluate_neural_simulator.py --output target/neural-simulator-audit.json",
        "component": "neural-simulator",
        "exit_code": 0,
        "failed": 0,
        "passed": 3
      }
    ],
    "model_and_decoder_gates_failed": 0,
    "model_and_decoder_gates_passed": 32,
    "protocol_id": "cyber-physical-software-regression-2026-08-20",
    "source_commit": "039ccd114f7b8cdacd17c609a3d84e0a1fe8d223"
  },
  "environment": {
    "accelerator": "WHPX",
    "host_os": "Windows 11",
    "qemu_version": "10.1.0",
    "queue_guest_ram_mb": 4096,
    "runtime_guest_ram_mb": 512
  },
  "global_limitations": [
    "The paper, physical simulator, neural synthetic, QEMU, and cyber-physical software evaluations use different protocols and are not directly comparable.",
    "No live EEG, human neural calibration, robot hardware-in-the-loop, or broad physical-PC benchmark is claimed.",
    "Ring-3 timing uses a virtualized 1 kHz PIT; provider, tool execution, and operator-confirmation latency are excluded.",
    "A passing benchmark is evidence for its named fixture and protocol, not formal safety proof."
  ],
  "metric_definitions": {
    "balanced_accuracy": {
      "definition": "Mean of sensitivity and specificity within the named fixture.",
      "higher_is_better": true,
      "range": [
        0.0,
        1.0
      ],
      "unit": "ratio"
    },
    "false_negative_rate": {
      "lower_is_better": true,
      "range": [
        0.0,
        1.0
      ],
      "unit": "ratio"
    },
    "false_positive_rate": {
      "lower_is_better": true,
      "range": [
        0.0,
        1.0
      ],
      "unit": "ratio"
    },
    "preview_latency": {
      "clock": "virtualized 1 kHz PIT with raw TSC retained in source runs",
      "lower_is_better": true,
      "unit": "microseconds"
    },
    "queue_batch_latency": {
      "lower_is_better": true,
      "request_count": 96,
      "unit": "milliseconds"
    }
  },
  "neural_synthetic": {
    "accepted_signal_accuracy": 1.0,
    "artifact_abstention_rate": 1.0,
    "artifact_trials": 400,
    "claim_boundary": [
      "Synthetic decoder evidence only; no human EEG accuracy or usability claim.",
      "No-control emitted intents are decoder candidates; OS commit additionally requires pairing, calibration, a local non-neural arm, signed preview, and revision checks.",
      "Physical neural intents remain proposal-only and cannot invoke an adapter."
    ],
    "emitted_intents": 0,
    "evidence_grade": "deterministic-synthetic-signal",
    "no_control_windows": 10000,
    "passed": true,
    "protocol_id": "neural-ssvep-synthetic-v1",
    "signal_trials": 600
  },
  "paired_preview_queue": {
    "baseline_batch_milliseconds": [
      15114.109,
      14910.766,
      15086.329
    ],
    "baseline_median_batch_milliseconds": 15086.329,
    "baseline_source_commit": "54e1cc1819d3b3c11436f437733d197d469d1607",
    "evidence_grade": "paired-repeated-emulator-measurement",
    "execution_records_added_each_run": [
      0,
      0,
      0
    ],
    "guest_fault_free_each_run": [
      true,
      true,
      true
    ],
    "limitation": "The single-threaded daemon serializes preview inference; concurrency here means multiple outstanding requests with response correlation, not parallel neural execution.",
    "median_improvement_percent": 10.4,
    "optimized_batch_milliseconds": [
      13784.147,
      13496.023,
      13517.414
    ],
    "optimized_median_batch_milliseconds": 13517.414,
    "optimized_median_milliseconds_per_serialized_request": 140.806,
    "optimized_source_commit": "c92056d8635af5e4ee2a81351350b1b25cfd4861",
    "protocol_id": "paired-preview-queue-whpx-2026-08-13",
    "requests_per_run": 96,
    "responses_received_each_run": [
      96,
      96,
      96
    ],
    "runs_each": 3
  },
  "paper_release": {
    "accepted_rows": 13697,
    "claim_boundary": "Authored balanced safety fixture; no material JEPA safety advantage over the per-action mean baseline was established.",
    "eligible_transitions": 13270,
    "episodes": 3639,
    "evidence_commit": "42ea7b8",
    "evidence_grade": "authored-balanced-fixture",
    "five_pipeline_balanced_accuracy_mean": 0.7976,
    "fixture_episodes": 500,
    "protocol_id": "world-model-study-v1.0.0/episode-disjoint-500",
    "release_tag": "world-model-study-v1.0.0",
    "rules_plus_jepa_balanced_accuracy": 0.8140000000000001,
    "rules_plus_jepa_false_negative_rate": 0.20799999999999996,
    "rules_plus_jepa_false_positive_rate": 0.16400000000000003,
    "rules_plus_mean_balanced_accuracy": 0.812
  },
  "physical_simulator_jepa": {
    "claim_boundary": "Deterministic simulator evidence; the artifact is permanently shadow-only and has no actuator authority.",
    "episodes": 2500,
    "evidence_grade": "deterministic-simulator",
    "false_negatives": 1,
    "false_positives": 16,
    "normalized_rollout_error": {
      "per_action_mean_h1": 0.019730109721422195,
      "per_action_mean_h2": 0.034468431025743484,
      "per_action_mean_h3": 0.04665490239858627,
      "per_action_mean_h4": 0.05753195658326149,
      "per_action_mean_h5": 0.06636197119951248,
      "physical_jepa_h1": 0.004541031550616026,
      "physical_jepa_h2": 0.007690879516303539,
      "physical_jepa_h3": 0.010492870584130287,
      "physical_jepa_h4": 0.01306567620486021,
      "physical_jepa_h5": 0.015461564064025879
    },
    "protocol_id": "physical-jepa-simulator-v1",
    "rules_plus_jepa_balanced_accuracy": 0.9944006975286777,
    "transitions": 15000,
    "validated_for_gating": false
  },
  "provenance": {
    "generator": "scripts/generate_public_evidence.py",
    "inputs": [
      {
        "path": "docs/benchmarks/raw/2026-08-13/manifest.json",
        "sha256": "030e41206f386e3318e06135fa5f2ce651948984ec39d7e4e212a30d9776b876"
      },
      {
        "path": "docs/benchmarks/raw/2026-08-13/runtime-run-1.json",
        "sha256": "699f099e6f87b203b08d84e848ee039e96d71126c21d4c94c7e5a223ea9a1523"
      },
      {
        "path": "docs/benchmarks/raw/2026-08-13/runtime-run-2.json",
        "sha256": "c857c0deb89d0afb0723016c2daf75d0607eda982a1e84bb283fcd4fa759aa26"
      },
      {
        "path": "docs/benchmarks/raw/2026-08-13/runtime-run-3.json",
        "sha256": "267db39b3ac2a59bd2502c3102680f7e2bfcd50395d919350700bf3e28c8c19b"
      },
      {
        "path": "docs/benchmarks/raw/2026-08-13/concurrency-baseline-run-1.json",
        "sha256": "11cde8b42902c57310e58d5da80259eed120ca3654e18cc5e96cca767c1aea01"
      },
      {
        "path": "docs/benchmarks/raw/2026-08-13/concurrency-baseline-run-2.json",
        "sha256": "2e507a785a5368f30c8ad823bd02d5229b69452f7ebc26ff421026b41131b369"
      },
      {
        "path": "docs/benchmarks/raw/2026-08-13/concurrency-baseline-run-3.json",
        "sha256": "668f162e9872294e4ed2dfd859448d01e7cce3f4a46c307676edd754a9d3b188"
      },
      {
        "path": "docs/benchmarks/raw/2026-08-13/concurrency-optimized-run-1.json",
        "sha256": "44ddaf62d43a3a73c40aca348aff175ec4e23ae01e75aa4be554650564bc8e35"
      },
      {
        "path": "docs/benchmarks/raw/2026-08-13/concurrency-optimized-run-2.json",
        "sha256": "bb8e12b0d3a8aa56bdc0af3903a6fdd74e134440295244369a777cc417b544db"
      },
      {
        "path": "docs/benchmarks/raw/2026-08-13/concurrency-optimized-run-3.json",
        "sha256": "6f9e0018844606136ec55b89d3da7ecb857bb4bb21f9cbc891a1b7a6456ac37a"
      },
      {
        "path": "docs/benchmarks/raw/2026-08-13/neural-synthetic.json",
        "sha256": "251996d01847f20f4ac17ae964c4b9ca72e27e0a89908407c6761c9a88b2bb79"
      },
      {
        "path": "docs/benchmarks/raw/2026-08-13/qemu-command-audit.json",
        "sha256": "374182fd18f13d3184e89c6ed4297aa54748ea4fb7b6495886bcb57c434830c2"
      },
      {
        "path": "docs/benchmarks/raw/2026-08-20/cyber-physical-software.json",
        "sha256": "9cadb74ee45c65d7bd19b97e4855f106a50834eca610b59dfe42b3254d06105c"
      },
      {
        "path": "docs/research/world_model_paper_evaluation.json",
        "sha256": "59e79803e6d3430fc3e6694cbf72ca7acb19f90b8cb5c335692ace7748ae6d58"
      },
      {
        "path": "docs/research/world_model_training_config.json",
        "sha256": "a6d0f3f198f28476c314951424de0114094fc8afc744c50b737c4d9aaed3604c"
      },
      {
        "path": "docs/research/physical_world_model_evaluation.json",
        "sha256": "a7c9e36f846750ca6988c7434fa4c34baffe35567eb68ab930534a8d6c6b7111"
      }
    ]
  },
  "qemu_command_audit": {
    "audit_source_commit": "45e369cececfc81e341c8772e295c4a00cba4bf1",
    "catalog_entries": 81,
    "catalog_passed": 81,
    "claim_boundary": "This is a dated emulator command-path audit for the named source commit and artifact hashes. It is not evidence of broad physical-PC compatibility, hardware accuracy, live EEG performance, formal safety, or independent replication.",
    "command_sweep_cases": 101,
    "command_sweep_passed": 101,
    "evidence_grade": "dated-emulator-measurement",
    "os_source_commit": "c92056d8635af5e4ee2a81351350b1b25cfd4861",
    "protocol_id": "ferrumos-qemu-command-audit-v1",
    "unknown_command_paths": 0
  },
  "schema_version": "2.1.0",
  "snapshot_date": "2026-08-20"
}