{
  "generated_at": "2026-09-17T09:38:32.549939+00:00",
  "n_emails": 2000,
  "seed": 20260916,
  "jev": {
    "n": 2000,
    "tp": 432,
    "fp": 180,
    "fn": 568,
    "tn": 820,
    "accuracy": 0.626,
    "accuracy_ci": [
      0.6045711453178447,
      0.6469457410468261
    ],
    "recall": 0.432,
    "recall_ci": [
      0.40161554299484964,
      0.4629049152128995
    ],
    "fpr": 0.18,
    "fpr_ci": [
      0.1574265005851786,
      0.20502271451011134
    ],
    "precision": 0.7058823529411765,
    "f1": 0.5359801488833746,
    "auroc": 0.6885385,
    "auroc_ci": [
      0.6666893911680324,
      0.7109252895727851
    ],
    "ece": 0.15361000000000002,
    "ece_ci": [
      0.13690475,
      0.17572287500000006
    ],
    "ece_bins": [
      {
        "lo": 0.5,
        "hi": 0.55,
        "n": 154,
        "accuracy": 0.512987012987013,
        "confidence": 0.5237012987012988
      },
      {
        "lo": 0.55,
        "hi": 0.6,
        "n": 168,
        "accuracy": 0.5357142857142857,
        "confidence": 0.5686309523809524
      },
      {
        "lo": 0.6,
        "hi": 0.65,
        "n": 179,
        "accuracy": 0.44692737430167595,
        "confidence": 0.621340782122905
      },
      {
        "lo": 0.65,
        "hi": 0.7,
        "n": 175,
        "accuracy": 0.5428571428571428,
        "confidence": 0.6704571428571428
      },
      {
        "lo": 0.7,
        "hi": 0.75,
        "n": 142,
        "accuracy": 0.6619718309859155,
        "confidence": 0.7192957746478874
      },
      {
        "lo": 0.75,
        "hi": 0.8,
        "n": 150,
        "accuracy": 0.6666666666666666,
        "confidence": 0.7705333333333334
      },
      {
        "lo": 0.8,
        "hi": 0.8500000000000001,
        "n": 217,
        "accuracy": 0.6543778801843319,
        "confidence": 0.8228571428571428
      },
      {
        "lo": 0.8500000000000001,
        "hi": 0.9,
        "n": 199,
        "accuracy": 0.5879396984924623,
        "confidence": 0.8761306532663317
      },
      {
        "lo": 0.9,
        "hi": 0.95,
        "n": 287,
        "accuracy": 0.6062717770034843,
        "confidence": 0.9214634146341465
      },
      {
        "lo": 0.95,
        "hi": 1.0,
        "n": 329,
        "accuracy": 0.8541033434650456,
        "confidence": 0.9708206686930091
      }
    ],
    "reliability": [
      {
        "lo": 0.0,
        "hi": 0.1,
        "n": 451,
        "observed": 0.3148558758314856,
        "predicted": 0.051330376940133045
      },
      {
        "lo": 0.1,
        "hi": 0.2,
        "n": 345,
        "observed": 0.4492753623188406,
        "predicted": 0.14142028985507246
      },
      {
        "lo": 0.2,
        "hi": 0.30000000000000004,
        "n": 224,
        "observed": 0.40625,
        "predicted": 0.2475
      },
      {
        "lo": 0.30000000000000004,
        "hi": 0.4,
        "n": 205,
        "observed": 0.4878048780487805,
        "predicted": 0.3502439024390244
      },
      {
        "lo": 0.4,
        "hi": 0.5,
        "n": 161,
        "observed": 0.4906832298136646,
        "predicted": 0.44484472049689433
      },
      {
        "lo": 0.5,
        "hi": 0.6000000000000001,
        "n": 192,
        "observed": 0.5208333333333334,
        "predicted": 0.5489583333333333
      },
      {
        "lo": 0.6000000000000001,
        "hi": 0.7000000000000001,
        "n": 130,
        "observed": 0.4846153846153846,
        "predicted": 0.655
      },
      {
        "lo": 0.7000000000000001,
        "hi": 0.8,
        "n": 84,
        "observed": 0.8452380952380952,
        "predicted": 0.7519047619047621
      },
      {
        "lo": 0.8,
        "hi": 0.9,
        "n": 80,
        "observed": 0.8875,
        "predicted": 0.845
      },
      {
        "lo": 0.9,
        "hi": 1.0,
        "n": 128,
        "observed": 1.0,
        "predicted": 0.9586718750000001
      }
    ],
    "brier": 0.2524997,
    "brier_ci": [
      0.24054301624999996,
      0.26579714875000005
    ],
    "f1_ci": [
      0.5051338993064604,
      0.5657819542988513
    ],
    "auto_decision": [
      {
        "threshold": 0.5,
        "coverage": 1.0,
        "n": 2000,
        "accuracy": 0.626,
        "errors": 748
      },
      {
        "threshold": 0.55,
        "coverage": 0.923,
        "n": 1846,
        "accuracy": 0.6354279523293608,
        "errors": 673
      },
      {
        "threshold": 0.6,
        "coverage": 0.839,
        "n": 1678,
        "accuracy": 0.6454112038140644,
        "errors": 595
      },
      {
        "threshold": 0.65,
        "coverage": 0.7495,
        "n": 1499,
        "accuracy": 0.6691127418278853,
        "errors": 496
      },
      {
        "threshold": 0.7,
        "coverage": 0.662,
        "n": 1324,
        "accuracy": 0.6858006042296072,
        "errors": 416
      },
      {
        "threshold": 0.75,
        "coverage": 0.591,
        "n": 1182,
        "accuracy": 0.688663282571912,
        "errors": 368
      },
      {
        "threshold": 0.8,
        "coverage": 0.516,
        "n": 1032,
        "accuracy": 0.6918604651162791,
        "errors": 318
      },
      {
        "threshold": 0.85,
        "coverage": 0.423,
        "n": 846,
        "accuracy": 0.7009456264775413,
        "errors": 253
      },
      {
        "threshold": 0.9,
        "coverage": 0.308,
        "n": 616,
        "accuracy": 0.7386363636363636,
        "errors": 161
      },
      {
        "threshold": 0.95,
        "coverage": 0.1645,
        "n": 329,
        "accuracy": 0.8541033434650456,
        "errors": 48
      },
      {
        "threshold": 0.98,
        "coverage": 0.0695,
        "n": 139,
        "accuracy": 0.9784172661870504,
        "errors": 3
      },
      {
        "threshold": 0.99,
        "coverage": 0.0375,
        "n": 75,
        "accuracy": 0.9866666666666667,
        "errors": 1
      }
    ],
    "prob_summary": {
      "mean_p_on_phishing": 0.45567,
      "mean_p_on_legit": 0.25915,
      "frac_extreme": 0.1235,
      "distinct_values": 101
    },
    "latency": {
      "n": 2000,
      "p50_s": 0.23926768700039247,
      "p95_s": 0.3307924170003389,
      "mean_s": 0.25164121410750706,
      "min_s": 0.20010270800048602,
      "max_s": 0.6433517919977021
    },
    "tokens": {
      "input": 1830888,
      "output": 450016,
      "calls": 2000
    }
  },
  "jev_model": "jev-1.13.0",
  "jev_errors": {
    "attempted": 2000,
    "api_errors": 0
  },
  "jev_cost": {
    "price_in_per_m": 0.042,
    "price_out_per_m": 0.0,
    "total_usd": 0.076897296,
    "per_1000_emails_usd": 0.038448648
  },
  "jev_internals": {
    "variants": {
      "verdict (choice)": {
        "accuracy": 0.626,
        "auroc": 0.6885385,
        "ece": 0.15361000000000002,
        "agreement": 1.0
      },
      "is_phishing (noul)": {
        "accuracy": 0.6055,
        "auroc": 0.683655,
        "ece": 0.17173499999999997,
        "agreement": 0.8315
      },
      "verdict_alt_click": {
        "accuracy": 0.632,
        "auroc": 0.6995275,
        "ece": 0.11499499999999999,
        "agreement": 0.801
      },
      "verdict_alt_minimal": {
        "accuracy": 0.5835,
        "auroc": 0.635026,
        "ece": 0.24125999999999997,
        "agreement": 0.7925
      }
    },
    "signals": {
      "sig_domain_mismatch": {
        "mean_phishing": 0.7521199999999999,
        "mean_legit": 0.3746900000000001,
        "auroc": 0.680337
      },
      "sig_free_hosting": {
        "mean_phishing": 0.86794,
        "mean_legit": 0.17389,
        "auroc": 0.959056
      },
      "sig_lure": {
        "mean_phishing": 0.6098300000000001,
        "mean_legit": 0.25169,
        "auroc": 0.8294455
      },
      "sig_urgency": {
        "mean_phishing": 0.04988000000000001,
        "mean_legit": 0.07061000000000002,
        "auroc": 0.3899395
      },
      "sig_generic_sender": {
        "mean_phishing": 0.49573,
        "mean_legit": 0.027880000000000002,
        "auroc": 0.935473
      }
    },
    "choice_noul_pearson": 0.9585862545451036,
    "conf_vs_maxp_pearson": 0.9997235445997998
  },
  "jev_composite": {
    "n": 2000,
    "rule_free_hosting": {
      "n": 2000,
      "tp": 884,
      "fp": 95,
      "fn": 116,
      "tn": 905,
      "accuracy": 0.8945,
      "accuracy_ci": [
        0.8802718782706878,
        0.9072155159425075
      ],
      "recall": 0.884,
      "recall_ci": [
        0.862666307122199,
        0.9023946347634532
      ],
      "fpr": 0.095,
      "fpr_ci": [
        0.07834495566471479,
        0.11475483219026154
      ],
      "precision": 0.9029622063329928,
      "f1": 0.8933804951995957
    },
    "cv_logistic": {
      "n": 2000,
      "tp": 961,
      "fp": 59,
      "fn": 39,
      "tn": 941,
      "accuracy": 0.951,
      "accuracy_ci": [
        0.9406441560750126,
        0.9596266038493247
      ],
      "recall": 0.961,
      "recall_ci": [
        0.9471303986994359,
        0.971341200803912
      ],
      "fpr": 0.059,
      "fpr_ci": [
        0.04601408784574914,
        0.07536123670744732
      ],
      "precision": 0.942156862745098,
      "f1": 0.9514851485148514,
      "auroc": 0.988384,
      "ece": 0.02670205668709308,
      "brier": 0.0398053748836187,
      "auto_decision": [
        {
          "threshold": 0.5,
          "coverage": 1.0,
          "n": 2000,
          "accuracy": 0.951,
          "errors": 98
        },
        {
          "threshold": 0.55,
          "coverage": 0.9885,
          "n": 1977,
          "accuracy": 0.952959028831563,
          "errors": 93
        },
        {
          "threshold": 0.6,
          "coverage": 0.969,
          "n": 1938,
          "accuracy": 0.9566563467492261,
          "errors": 84
        },
        {
          "threshold": 0.65,
          "coverage": 0.95,
          "n": 1900,
          "accuracy": 0.9605263157894737,
          "errors": 75
        },
        {
          "threshold": 0.7,
          "coverage": 0.933,
          "n": 1866,
          "accuracy": 0.9640943193997856,
          "errors": 67
        },
        {
          "threshold": 0.75,
          "coverage": 0.921,
          "n": 1842,
          "accuracy": 0.9690553745928339,
          "errors": 57
        },
        {
          "threshold": 0.8,
          "coverage": 0.9025,
          "n": 1805,
          "accuracy": 0.978393351800554,
          "errors": 39
        },
        {
          "threshold": 0.85,
          "coverage": 0.8845,
          "n": 1769,
          "accuracy": 0.9858677218767665,
          "errors": 25
        },
        {
          "threshold": 0.9,
          "coverage": 0.861,
          "n": 1722,
          "accuracy": 0.9901277584204413,
          "errors": 17
        },
        {
          "threshold": 0.95,
          "coverage": 0.8015,
          "n": 1603,
          "accuracy": 0.9943855271366189,
          "errors": 9
        },
        {
          "threshold": 0.98,
          "coverage": 0.6385,
          "n": 1277,
          "accuracy": 0.9937353171495693,
          "errors": 8
        },
        {
          "threshold": 0.99,
          "coverage": 0.4645,
          "n": 929,
          "accuracy": 0.9935414424111948,
          "errors": 6
        }
      ],
      "weights_full_fit": {
        "bias": -5.0440364148382235,
        "sig_domain_mismatch": -4.930360390206134,
        "sig_free_hosting": 9.270147439514394,
        "sig_lure": 0.9972859202855212,
        "sig_urgency": -0.771919455986489,
        "sig_generic_sender": 12.238083661606598,
        "verdict_p": 3.5308062300162444
      }
    }
  },
  "stability": {
    "Jev pass 1 vs pass 2 (choice p)": {
      "n": 2000,
      "mean_abs_diff": 0.01708,
      "p95_abs_diff": 0.05000000000000002,
      "max_abs_diff": 0.15000000000000008,
      "frac_diff_over_0_05": 0.052,
      "identical_frac": 0.259,
      "label_flip_rate": 0.022,
      "pearson_r": 0.9963165243840985
    },
    "Jev pass 1 vs pass 3 (choice p, 12 h later)": {
      "n": 200,
      "mean_abs_diff": 0.0176,
      "p95_abs_diff": 0.050000000000000044,
      "max_abs_diff": 0.07999999999999996,
      "frac_diff_over_0_05": 0.065,
      "identical_frac": 0.235,
      "label_flip_rate": 0.01,
      "pearson_r": 0.9964854734255344
    },
    "Jev pass 1 vs pass 2 (noul)": {
      "n": 2000,
      "mean_abs_diff": 0.010625,
      "p95_abs_diff": 0.030000000000000027,
      "max_abs_diff": 0.10000000000000003,
      "frac_diff_over_0_05": 0.01,
      "identical_frac": 0.3545,
      "label_flip_rate": 0.022,
      "pearson_r": 0.9970004284418379
    },
    "claude-haiku-4-5 pass 1 vs pass 2": {
      "n": 300,
      "mean_abs_diff": 0.0053999999999999986,
      "p95_abs_diff": 0.0,
      "max_abs_diff": 0.6499999999999999,
      "frac_diff_over_0_05": 0.016666666666666666,
      "identical_frac": 0.98,
      "label_flip_rate": 0.006666666666666667,
      "pearson_r": 0.9894488608585359
    }
  },
  "by_category": {
    "jev": {
      "url_shortener": {
        "n": 154,
        "correct": 100,
        "y": 1,
        "accuracy": 0.6493506493506493,
        "accuracy_ci": [
          0.5711844194887772,
          0.7202469902283062
        ]
      },
      "cross_domain_legitimate": {
        "n": 333,
        "correct": 183,
        "y": 0,
        "accuracy": 0.5495495495495496,
        "accuracy_ci": [
          0.4958477322401643,
          0.6021211647131811
        ]
      },
      "hosting_platform": {
        "n": 17,
        "correct": 17,
        "y": 1,
        "accuracy": 1.0,
        "accuracy_ci": [
          0.8156763396284354,
          1.0
        ]
      },
      "google_docs": {
        "n": 196,
        "correct": 3,
        "y": 1,
        "accuracy": 0.015306122448979591,
        "accuracy_ci": [
          0.005218785274458761,
          0.04402821835241372
        ]
      },
      "github_pages": {
        "n": 227,
        "correct": 39,
        "y": 1,
        "accuracy": 0.17180616740088106,
        "accuracy_ci": [
          0.12830094808041426,
          0.22623480284142994
        ]
      },
      "legitimate": {
        "n": 667,
        "correct": 637,
        "y": 0,
        "accuracy": 0.9550224887556222,
        "accuracy_ci": [
          0.9365180281835451,
          0.9683155420661235
        ]
      },
      "firebase": {
        "n": 101,
        "correct": 46,
        "y": 1,
        "accuracy": 0.45544554455445546,
        "accuracy_ci": [
          0.36173444762795837,
          0.5524217651730671
        ]
      },
      "short_clean_https": {
        "n": 190,
        "correct": 158,
        "y": 1,
        "accuracy": 0.8315789473684211,
        "accuracy_ci": [
          0.7719149408997723,
          0.8781003293002261
        ]
      },
      "ipfs": {
        "n": 102,
        "correct": 62,
        "y": 1,
        "accuracy": 0.6078431372549019,
        "accuracy_ci": [
          0.5108314489360125,
          0.6970263309917285
        ]
      },
      "other_https": {
        "n": 13,
        "correct": 7,
        "y": 1,
        "accuracy": 0.5384615384615384,
        "accuracy_ci": [
          0.2914343243183205,
          0.7679424569851186
        ]
      }
    },
    "llm": {
      "url_shortener": {
        "n": 154,
        "correct": 153,
        "y": 1,
        "accuracy": 0.9935064935064936,
        "accuracy_ci": [
          0.9641379001160711,
          0.9988528450360307
        ]
      },
      "cross_domain_legitimate": {
        "n": 333,
        "correct": 289,
        "y": 0,
        "accuracy": 0.8678678678678678,
        "accuracy_ci": [
          0.8272660573211691,
          0.9000787836960925
        ]
      },
      "hosting_platform": {
        "n": 17,
        "correct": 17,
        "y": 1,
        "accuracy": 1.0,
        "accuracy_ci": [
          0.8156763396284354,
          1.0
        ]
      },
      "google_docs": {
        "n": 196,
        "correct": 6,
        "y": 1,
        "accuracy": 0.030612244897959183,
        "accuracy_ci": [
          0.014103666502553219,
          0.06516711595715485
        ]
      },
      "github_pages": {
        "n": 227,
        "correct": 192,
        "y": 1,
        "accuracy": 0.8458149779735683,
        "accuracy_ci": [
          0.7931196490670536,
          0.8870003986193253
        ]
      },
      "legitimate": {
        "n": 667,
        "correct": 573,
        "y": 0,
        "accuracy": 0.8590704647676162,
        "accuracy_ci": [
          0.8306034237550491,
          0.8834250443661883
        ]
      },
      "firebase": {
        "n": 101,
        "correct": 101,
        "y": 1,
        "accuracy": 1.0,
        "accuracy_ci": [
          0.9633580563440465,
          0.9999999999999998
        ]
      },
      "short_clean_https": {
        "n": 190,
        "correct": 180,
        "y": 1,
        "accuracy": 0.9473684210526315,
        "accuracy_ci": [
          0.9058408551834182,
          0.9711638744515
        ]
      },
      "ipfs": {
        "n": 102,
        "correct": 102,
        "y": 1,
        "accuracy": 1.0,
        "accuracy_ci": [
          0.9637042523922542,
          1.0000000000000002
        ]
      },
      "other_https": {
        "n": 13,
        "correct": 13,
        "y": 1,
        "accuracy": 1.0,
        "accuracy_ci": [
          0.7718981569447083,
          1.0
        ]
      }
    }
  },
  "controls": {
    "split": {
      "seed": 20260917,
      "n_a": 1000,
      "n_b": 1000,
      "phishing_a": 500,
      "phishing_b": 500
    },
    "heuristic_all": {
      "hosting_or_shortener": {
        "n": 2000,
        "tp": 835,
        "fp": 2,
        "fn": 165,
        "tn": 998,
        "accuracy": 0.9165,
        "accuracy_ci": [
          0.9035627287371036,
          0.9278403123016691
        ],
        "recall": 0.835,
        "recall_ci": [
          0.8107202824572156,
          0.8567156954899028
        ],
        "fpr": 0.002,
        "fpr_ci": [
          0.000548632102457155,
          0.007262958889587807
        ],
        "precision": 0.997610513739546,
        "f1": 0.9090909090909091,
        "auroc": 0.9165
      },
      "etld1_mismatch": {
        "n": 2000,
        "tp": 931,
        "fp": 346,
        "fn": 69,
        "tn": 654,
        "accuracy": 0.7925,
        "accuracy_ci": [
          0.7741748839600691,
          0.8097036022436308
        ],
        "recall": 0.931,
        "recall_ci": [
          0.9135849276398866,
          0.945116285778645
        ],
        "fpr": 0.346,
        "fpr_ci": [
          0.31715614268933207,
          0.37602254207527624
        ],
        "precision": 0.7290524667188724,
        "f1": 0.8177426438296004,
        "auroc": 0.7925
      },
      "either": {
        "n": 2000,
        "tp": 932,
        "fp": 348,
        "fn": 68,
        "tn": 652,
        "accuracy": 0.792,
        "accuracy_ci": [
          0.7736601100068136,
          0.8092202933144871
        ],
        "recall": 0.932,
        "recall_ci": [
          0.9146857880858408,
          0.946007771535518
        ],
        "fpr": 0.348,
        "fpr_ci": [
          0.3191088817410635,
          0.3780544954291993
        ],
        "precision": 0.728125,
        "f1": 0.8175438596491228
      },
      "both": {
        "n": 2000,
        "tp": 834,
        "fp": 0,
        "fn": 166,
        "tn": 1000,
        "accuracy": 0.917,
        "accuracy_ci": [
          0.9040946609667939,
          0.9283064629543785
        ],
        "recall": 0.834,
        "recall_ci": [
          0.809668727086498,
          0.8557749046577932
        ],
        "fpr": 0.0,
        "fpr_ci": [
          0.0,
          0.003826898586390522
        ],
        "precision": 1.0,
        "f1": 0.9094874591057798
      },
      "ordinal_score_auroc": 0.937192,
      "ordinal_score_auroc_ci": [
        0.9269471151152909,
        0.9478696531007061
      ]
    },
    "heuristic_split": {
      "n_a": 1000,
      "n_b": 1000,
      "features": [
        "hosting_or_shortener",
        "etld1_mismatch"
      ],
      "single_rule": {
        "n": 1000,
        "tp": 419,
        "fp": 1,
        "fn": 81,
        "tn": 499,
        "accuracy": 0.918,
        "accuracy_ci": [
          0.8993524099554584,
          0.9334483028263192
        ],
        "recall": 0.838,
        "recall_ci": [
          0.8031470479440461,
          0.8676987099290632
        ],
        "fpr": 0.002,
        "fpr_ci": [
          0.000353127309584542,
          0.011240992747195207
        ],
        "precision": 0.9976190476190476,
        "f1": 0.9108695652173913,
        "feature": "hosting_or_shortener",
        "threshold": 0.5,
        "auroc_a_all_features": {
          "hosting_or_shortener": 0.915,
          "etld1_mismatch": 0.799
        },
        "auroc_b": 0.918,
        "auroc_b_ci": [
          0.9002016303563699,
          0.9341597401590486
        ],
        "accuracy_a": 0.915
      },
      "logistic": {
        "n": 1000,
        "tp": 419,
        "fp": 1,
        "fn": 81,
        "tn": 499,
        "accuracy": 0.918,
        "accuracy_ci": [
          0.8993524099554584,
          0.9334483028263192
        ],
        "recall": 0.838,
        "recall_ci": [
          0.8031470479440461,
          0.8676987099290632
        ],
        "fpr": 0.002,
        "fpr_ci": [
          0.000353127309584542,
          0.011240992747195207
        ],
        "precision": 0.9976190476190476,
        "f1": 0.9108695652173913,
        "auroc": 0.93742,
        "auroc_ci": [
          0.9217068803074454,
          0.9518578118593918
        ],
        "ece": 0.006239241039093132,
        "brier": 0.06895959575296413,
        "weights": {
          "bias": -2.2861814482452587,
          "hosting_or_shortener": 7.157554422828563,
          "etld1_mismatch": 1.0995217302616915
        },
        "accuracy_a": 0.915
      }
    },
    "jev_signals_split": {
      "n_a": 1000,
      "n_b": 1000,
      "features": [
        "sig_domain_mismatch",
        "sig_free_hosting",
        "sig_lure",
        "sig_urgency",
        "sig_generic_sender"
      ],
      "single_rule": {
        "n": 1000,
        "tp": 432,
        "fp": 38,
        "fn": 68,
        "tn": 462,
        "accuracy": 0.894,
        "accuracy_ci": [
          0.8733891741189649,
          0.9115952297949595
        ],
        "recall": 0.864,
        "recall_ci": [
          0.8311642695066905,
          0.8912850082028119
        ],
        "fpr": 0.076,
        "fpr_ci": [
          0.05586871073410857,
          0.10259696578009747
        ],
        "precision": 0.9191489361702128,
        "f1": 0.8907216494845361,
        "feature": "sig_free_hosting",
        "threshold": 0.705,
        "auroc_a_all_features": {
          "sig_domain_mismatch": 0.686106,
          "sig_free_hosting": 0.959982,
          "sig_lure": 0.8426,
          "sig_urgency": 0.389868,
          "sig_generic_sender": 0.940752
        },
        "auroc_b": 0.958288,
        "auroc_b_ci": [
          0.9476161080632266,
          0.9685026488368891
        ],
        "accuracy_a": 0.901
      },
      "logistic": {
        "n": 1000,
        "tp": 485,
        "fp": 35,
        "fn": 15,
        "tn": 465,
        "accuracy": 0.95,
        "accuracy_ci": [
          0.9346858639155304,
          0.9618699273567182
        ],
        "recall": 0.97,
        "recall_ci": [
          0.9510959078150344,
          0.9817369507659165
        ],
        "fpr": 0.07,
        "fpr_ci": [
          0.05075944317125315,
          0.09579772876532376
        ],
        "precision": 0.9326923076923077,
        "f1": 0.9509803921568627,
        "auroc": 0.982488,
        "auroc_ci": [
          0.9753059804693509,
          0.9889642799325326
        ],
        "ece": 0.024320483893187198,
        "brier": 0.043959664187745204,
        "weights": {
          "bias": -4.595841802779937,
          "sig_domain_mismatch": -6.199542960823362,
          "sig_free_hosting": 10.619859967092172,
          "sig_lure": 3.9644110168540325,
          "sig_urgency": 1.329979941913166,
          "sig_generic_sender": 10.72566706731017
        },
        "accuracy_a": 0.959
      }
    },
    "llm_signals_split": {
      "n_a": 999,
      "n_b": 1000,
      "features": [
        "sig_domain_mismatch",
        "sig_free_hosting",
        "sig_lure",
        "sig_urgency",
        "sig_generic_sender"
      ],
      "single_rule": {
        "n": 1000,
        "tp": 444,
        "fp": 2,
        "fn": 56,
        "tn": 498,
        "accuracy": 0.942,
        "accuracy_ci": [
          0.9257500917212245,
          0.9548669299284063
        ],
        "recall": 0.888,
        "recall_ci": [
          0.8573456933034787,
          0.9127376027165404
        ],
        "fpr": 0.004,
        "fpr_ci": [
          0.0010976075412372681,
          0.014466014041418871
        ],
        "precision": 0.9955156950672646,
        "f1": 0.9386892177589852,
        "feature": "sig_generic_sender",
        "threshold": 0.07500000000000001,
        "auroc_a_all_features": {
          "sig_domain_mismatch": 0.9232625250501002,
          "sig_free_hosting": 0.9347074148296594,
          "sig_lure": 0.8674068136272545,
          "sig_urgency": 0.8015290581162324,
          "sig_generic_sender": 0.9556452905811623
        },
        "auroc_b": 0.949028,
        "auroc_b_ci": [
          0.935323042698931,
          0.962050735982114
        ],
        "accuracy_a": 0.9479479479479479
      },
      "logistic": {
        "n": 1000,
        "tp": 459,
        "fp": 27,
        "fn": 41,
        "tn": 473,
        "accuracy": 0.932,
        "accuracy_ci": [
          0.9146857880858408,
          0.946007771535518
        ],
        "recall": 0.918,
        "recall_ci": [
          0.8906445762331059,
          0.938981242675059
        ],
        "fpr": 0.054,
        "fpr_ci": [
          0.03737409848513375,
          0.07742706124443205
        ],
        "precision": 0.9444444444444444,
        "f1": 0.9310344827586207,
        "auroc": 0.9911,
        "auroc_ci": [
          0.9871188617392622,
          0.994431454824135
        ],
        "ece": 0.030816338819275983,
        "brier": 0.036643579987527264,
        "weights": {
          "bias": -5.3362725361869785,
          "sig_domain_mismatch": -2.3032392790201563,
          "sig_free_hosting": 1.1962560285849353,
          "sig_lure": 6.8522830204159435,
          "sig_urgency": 3.93737853836159,
          "sig_generic_sender": 13.152749437056059
        },
        "accuracy_a": 0.953953953953954
      },
      "run": {
        "model": "claude-haiku-4-5",
        "attempted": 2000,
        "api_errors": 1,
        "format_errors": 0,
        "fenced": 1999,
        "latency": {
          "n": 1999,
          "p50_s": 1.1992970420033089,
          "p95_s": 1.53684995900403,
          "mean_s": 1.1921805014832483,
          "min_s": 0.7183601249998901,
          "max_s": 5.755836874996021
        },
        "tokens_per_email": {
          "input": 616.1985992996498,
          "output": 80.0
        },
        "cost_per_1000_usd": 1.0161985992996498,
        "signal_means": {
          "sig_domain_mismatch": {
            "phishing": 0.8245999999999998,
            "legit": 0.2132132132132132
          },
          "sig_free_hosting": {
            "phishing": 0.82877,
            "legit": 0.15335335335335337
          },
          "sig_lure": {
            "phishing": 0.53625,
            "legit": 0.22771771771771773
          },
          "sig_urgency": {
            "phishing": 0.12595,
            "legit": 0.051201201201201196
          },
          "sig_generic_sender": {
            "phishing": 0.72001,
            "legit": 0.0024024024024024023
          }
        }
      }
    },
    "paired_b": {
      "jev_logistic_vs_heuristic_logistic": {
        "n": 1000,
        "only_first_correct": 66,
        "only_second_correct": 34,
        "mcnemar_p": 0.0017899303914868526
      },
      "jev_rule_vs_heuristic_rule": {
        "n": 1000,
        "only_first_correct": 19,
        "only_second_correct": 43,
        "mcnemar_p": 0.0031560602510210594
      },
      "jev_logistic_vs_llm_logistic": {
        "n": 1000,
        "only_first_correct": 51,
        "only_second_correct": 33,
        "mcnemar_p": 0.06297225703616971
      },
      "jev_rule_vs_llm_rule": {
        "n": 1000,
        "only_first_correct": 34,
        "only_second_correct": 82,
        "mcnemar_p": 9.685884063140465e-06
      },
      "llm_logistic_vs_heuristic_logistic": {
        "n": 1000,
        "only_first_correct": 65,
        "only_second_correct": 51,
        "mcnemar_p": 0.2272749463316086
      }
    }
  },
  "net_floor": {
    "typesafe_warm_p50_s": 0.1629911460004223,
    "typesafe_cold_p50_s": 0.5294340624986944,
    "llm_warm_p50_s": 0.018402457999400212,
    "llm_cold_p50_s": 0.07907781200083264,
    "measured_at": "2026-09-16T21:37:01.586077+00:00"
  },
  "llm_model": "claude-haiku-4-5",
  "llm": {
    "n": 2000,
    "tp": 764,
    "fp": 138,
    "fn": 236,
    "tn": 862,
    "accuracy": 0.813,
    "accuracy_ci": [
      0.7953171621099848,
      0.829482722272094
    ],
    "recall": 0.764,
    "recall_ci": [
      0.7367023285873138,
      0.789277068959072
    ],
    "fpr": 0.138,
    "fpr_ci": [
      0.11800423096411534,
      0.16076644361243145
    ],
    "precision": 0.8470066518847007,
    "f1": 0.8033648790746583,
    "auroc": 0.836796,
    "auroc_ci": [
      0.8202398612361826,
      0.853160988889119
    ],
    "ece": 0.09714000000000005,
    "ece_ci": [
      0.0808415,
      0.11546762500000002
    ],
    "ece_bins": [
      {
        "lo": 0.5,
        "hi": 0.55,
        "n": 0
      },
      {
        "lo": 0.55,
        "hi": 0.6,
        "n": 0
      },
      {
        "lo": 0.6,
        "hi": 0.65,
        "n": 15,
        "accuracy": 1.0,
        "confidence": 0.5999999999999999
      },
      {
        "lo": 0.65,
        "hi": 0.7,
        "n": 0
      },
      {
        "lo": 0.7,
        "hi": 0.75,
        "n": 249,
        "accuracy": 0.7831325301204819,
        "confidence": 0.7054618473895581
      },
      {
        "lo": 0.75,
        "hi": 0.8,
        "n": 223,
        "accuracy": 0.7982062780269058,
        "confidence": 0.75
      },
      {
        "lo": 0.8,
        "hi": 0.8500000000000001,
        "n": 405,
        "accuracy": 0.8,
        "confidence": 0.8499999999999998
      },
      {
        "lo": 0.8500000000000001,
        "hi": 0.9,
        "n": 0
      },
      {
        "lo": 0.9,
        "hi": 0.95,
        "n": 24,
        "accuracy": 0.9166666666666666,
        "confidence": 0.92
      },
      {
        "lo": 0.95,
        "hi": 1.0,
        "n": 1084,
        "accuracy": 0.8228782287822878,
        "confidence": 0.9500553505535057
      }
    ],
    "reliability": [
      {
        "lo": 0.0,
        "hi": 0.1,
        "n": 996,
        "observed": 0.1927710843373494,
        "predicted": 0.05
      },
      {
        "lo": 0.1,
        "hi": 0.2,
        "n": 102,
        "observed": 0.43137254901960786,
        "predicted": 0.15
      },
      {
        "lo": 0.2,
        "hi": 0.30000000000000004,
        "n": 0
      },
      {
        "lo": 0.30000000000000004,
        "hi": 0.4,
        "n": 0
      },
      {
        "lo": 0.4,
        "hi": 0.5,
        "n": 0
      },
      {
        "lo": 0.5,
        "hi": 0.6000000000000001,
        "n": 15,
        "observed": 1.0,
        "predicted": 0.5999999999999999
      },
      {
        "lo": 0.6000000000000001,
        "hi": 0.7000000000000001,
        "n": 181,
        "observed": 0.8232044198895028,
        "predicted": 0.7000000000000002
      },
      {
        "lo": 0.7000000000000001,
        "hi": 0.8,
        "n": 291,
        "observed": 0.7697594501718213,
        "predicted": 0.7429896907216494
      },
      {
        "lo": 0.8,
        "hi": 0.9,
        "n": 303,
        "observed": 0.8778877887788779,
        "predicted": 0.8499999999999996
      },
      {
        "lo": 0.9,
        "hi": 1.0,
        "n": 112,
        "observed": 0.9821428571428571,
        "predicted": 0.9441071428571427
      }
    ],
    "brier": 0.1630453,
    "brier_ci": [
      0.1510147975,
      0.17588782624999996
    ],
    "f1_ci": [
      0.7829776894444189,
      0.822187454426443
    ],
    "auto_decision": [
      {
        "threshold": 0.5,
        "coverage": 1.0,
        "n": 2000,
        "accuracy": 0.813,
        "errors": 374
      },
      {
        "threshold": 0.55,
        "coverage": 1.0,
        "n": 2000,
        "accuracy": 0.813,
        "errors": 374
      },
      {
        "threshold": 0.6,
        "coverage": 1.0,
        "n": 2000,
        "accuracy": 0.813,
        "errors": 374
      },
      {
        "threshold": 0.65,
        "coverage": 0.9925,
        "n": 1985,
        "accuracy": 0.8115869017632242,
        "errors": 374
      },
      {
        "threshold": 0.7,
        "coverage": 0.9925,
        "n": 1985,
        "accuracy": 0.8115869017632242,
        "errors": 374
      },
      {
        "threshold": 0.75,
        "coverage": 0.868,
        "n": 1736,
        "accuracy": 0.815668202764977,
        "errors": 320
      },
      {
        "threshold": 0.8,
        "coverage": 0.7565,
        "n": 1513,
        "accuracy": 0.8182419035029742,
        "errors": 275
      },
      {
        "threshold": 0.85,
        "coverage": 0.7565,
        "n": 1513,
        "accuracy": 0.8182419035029742,
        "errors": 275
      },
      {
        "threshold": 0.9,
        "coverage": 0.554,
        "n": 1108,
        "accuracy": 0.8249097472924187,
        "errors": 194
      },
      {
        "threshold": 0.95,
        "coverage": 0.542,
        "n": 1084,
        "accuracy": 0.8228782287822878,
        "errors": 192
      },
      {
        "threshold": 0.98,
        "coverage": 0.001,
        "n": 2,
        "accuracy": 1.0,
        "errors": 0
      },
      {
        "threshold": 0.99,
        "coverage": 0.0,
        "n": 0,
        "accuracy": NaN,
        "errors": 0
      }
    ],
    "prob_summary": {
      "mean_p_on_phishing": 0.6261199999999999,
      "mean_p_on_legit": 0.15417999999999998,
      "frac_extreme": 0.001,
      "distinct_values": 10
    },
    "latency": {
      "n": 2000,
      "p50_s": 0.6871161249982833,
      "p95_s": 0.9800936669998919,
      "mean_s": 0.7358708065094889,
      "min_s": 0.44639166699926136,
      "max_s": 6.127803125000355
    },
    "tokens": {
      "input": 674363,
      "output": 50000,
      "calls": 2000
    }
  },
  "llm_errors": {
    "api_errors": 0,
    "format_errors": 0,
    "fenced": 2000,
    "attempted": 2000,
    "model": "claude-haiku-4-5"
  },
  "llm_cost": {
    "price_in_per_m": 1.0,
    "price_out_per_m": 5.0,
    "total_usd": 0.924363,
    "per_1000_emails_usd": 0.4621815
  },
  "reference": {
    "model": "anthropic/claude-haiku-4.5",
    "recall": 0.956,
    "fpr": 0.799,
    "safetility": 0.0
  },
  "ratios": {
    "paired_n": 2000,
    "jev_only_correct": 85,
    "llm_only_correct": 459,
    "mcnemar_p": 4.954011946448274e-63,
    "latency_p50": 2.871746425990051,
    "latency_p95": 2.9628661862550727,
    "cost": 12.020747777659178
  }
}