"""Opt-in outbound notification webhook (the ONLY new outbound egress path).
On-call gets pinged when a high-confidence true-positive lands, a hunt turns up a
threat finding, or the analyst model grades UNFIT. Everything else in soc-ai is
zero-egress; this feature is the one exception, so it is treated like the Oracle:
* DEFAULT OFF. :func:`fire` returns BEFORE constructing any HTTP client unless
``notify_enabled`` is on, the relevant per-trigger toggle is on, and a
``notify_webhook_url`` is configured. When off, NO ``httpx`` call is ever
made — proven by ``tests/test_notify.py``.
* The webhook URL is a SECRET (``SecretStr``; Fernet-encrypted at rest, never
rendered back through the config console).
* Every attempted send is AUDITED (``"notification"`` audit kind), best-effort
— an audit failure never breaks the send, and a webhook failure never breaks
the investigation/hunt that triggered it (fail-soft, log + swallow).
The payload shape is selected by ``notify_format``:
* ``json`` — a compact generic dict ``{kind,title,body,url,severity}``.
* ``slack`` — ``{"text": "
— "}``.
* ``matrix`` — ``{"msgtype":"m.text","body":" — "}``.
Sends are deduped in-process by ``(kind, entity, current-hour)`` so a re-fired
trigger inside the same hour posts once. The dedup is a bounded best-effort LRU —
not durable across a restart (that is acceptable: a missed dedup at most double-
pings on-call once).
"""
from __future__ import annotations
import logging
from collections import OrderedDict
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import Any
import httpx
from soc_ai.demo.guard import assert_egress_allowed
from soc_ai.hunting.outcome import FindingVerdicts, outcome_label
_LOGGER = logging.getLogger(__name__)
# Valid ``notify_format`` values (kept here so config.py + the route + tests share
# one source of truth).
NOTIFY_FORMATS: tuple[str, ...] = ("json", "slack", "matrix")
# Trigger kinds a NotifyEvent may carry. ``test`` is the canned validation event
# fired by the Test button — it bypasses the per-trigger toggles (below) but still
# requires a configured webhook URL. ``quality_regression`` is fired by the
# ``soc-ai eval-nightly`` CLI (not the server) when the nightly micro-eval trips
# its regression detector.
NOTIFY_KINDS: tuple[str, ...] = (
"tp",
"hunt_threat",
"model_fitness_fail",
"quality_regression",
"audit_chain_break",
"tls_expiry",
"oracle_paused",
"test",
)
# The tone of each hunt outcome on the bell and the Notifications screen. A
# hunt that found a threat reads like a true positive. A hunt that could not run
# reads warn, because its result is unknown and somebody has to look. The bell
# said "Hunt finished: no threat observed" for 8 analytic runs that could not
# reach the data source. A hunt with no answer reads warn for the same reason.
# The two clean outcomes and a running hunt are informational. The webhook fires
# on a threat only (:func:`event_for_hunt`).
HUNT_OUTCOME_TONE: dict[str, str] = {
"threat_found": "danger",
"could_not_run": "warn",
# The hunt ran and did not test its objective, so the analyst has to act.
"no_answer": "warn",
"no_threat_gap": "accent",
"no_threat": "accent",
"running": "accent",
# The investigations of the hunt's findings decided. A true positive among
# them reads like a true positive.
"threats_confirmed": "danger",
"no_threat_confirmed": "accent",
"investigating": "accent",
}
def hunt_outcome_tone(outcome: str) -> str:
"""The bell tone of a hunt outcome. An unknown outcome reads warn.
An outcome a newer build adds must not read as a quiet, clean result.
"""
return HUNT_OUTCOME_TONE.get(outcome, "warn")
# Per-send bound. Short: a webhook is fire-and-forget on-call plumbing, not on the
# investigation's critical path — a slow endpoint must not stall the trigger.
_SEND_TIMEOUT_S = 5.0
# Retries AFTER the first attempt (so 2 = up to 3 total). Only transient failures
# (connect/read/5xx) are retried; a 4xx is terminal (misconfigured webhook).
_MAX_RETRIES = 2
_RETRY_BACKOFF_S = 0.5
# In-process dedup ring. Bounded so a long-running process can't leak memory: an
# OrderedDict used as an LRU set (value is unused). Best-effort only.
_DEDUP_MAX = 512
_dedup_seen: OrderedDict[str, None] = OrderedDict()
@dataclass
class NotifyEvent:
"""One thing worth pinging on-call about.
``kind`` is one of :data:`NOTIFY_KINDS`. ``url`` is a permalink into the app
(e.g. ``/app/investigation/``) so the recipient can jump straight to it.
``severity`` is advisory (``info``/``warning``/``critical``) — it rides in the
generic-JSON payload but is not otherwise load-bearing.
"""
kind: str
title: str
body: str
url: str
severity: str = "info"
def _entity_key(event: NotifyEvent) -> str:
"""A stable per-entity dedup key for an event.
The permalink URL uniquely identifies the entity (an investigation id, a hunt
id) for the tp / hunt_threat kinds; model-fitness has no permalink entity, so
its title (which names the model + grade) is the stable key. Falls back to the
title when the URL is empty.
"""
return event.url.strip() or event.title.strip()
def _dedup_should_send(event: NotifyEvent, *, now: datetime | None = None) -> bool:
"""True iff this (kind, entity, hour) hasn't been sent this hour.
Records the key when it returns True. Best-effort + bounded (an LRU set): a
process restart clears it, so at most one duplicate ping can slip through per
restart. Never raises.
"""
now = now or datetime.now(UTC)
hour = now.strftime("%Y%m%dT%H")
key = f"{event.kind}:{_entity_key(event)}:{hour}"
if key in _dedup_seen:
# Refresh recency so a repeatedly-fired key stays warm in the LRU.
_dedup_seen.move_to_end(key)
return False
_dedup_seen[key] = None
while len(_dedup_seen) > _DEDUP_MAX:
_dedup_seen.popitem(last=False)
return True
def _build_payload(event: NotifyEvent, fmt: str) -> dict[str, Any]:
"""Render *event* into the wire body for *fmt*.
``slack``/``matrix`` collapse to a single human line; ``json`` (and any
unknown format, defensively) is the generic structured dict.
"""
line = f"{event.title}. {event.body}"
if event.url:
line = f"{line} {event.url}"
if fmt == "slack":
return {"text": line}
if fmt == "matrix":
return {"msgtype": "m.text", "body": line}
# json (default) — a compact structured dict.
return {
"kind": event.kind,
"title": event.title,
"body": event.body,
"url": event.url,
"severity": event.severity,
}
def _webhook_url(settings: Any) -> str:
"""The configured webhook URL as a plain string, or '' when unset.
``notify_webhook_url`` is a ``SecretStr | None``; unwrap it. Never logged.
"""
secret = getattr(settings, "notify_webhook_url", None)
if secret is None:
return ""
try:
return str(secret.get_secret_value()).strip()
except AttributeError:
# A test double may hand us a bare string.
return str(secret).strip()
def _trigger_enabled(settings: Any, kind: str) -> bool:
"""Whether the per-trigger toggle for *kind* is on.
The ``test`` kind bypasses the per-trigger toggles (it's an explicit operator
validation, gated only on enabled+url by the caller). All others map to their
``notify_on_*`` flag; an unknown kind is refused (no toggle → no send).
"""
if kind == "test":
return True
flag = {
"tp": "notify_on_tp",
"hunt_threat": "notify_on_hunt_threat",
"model_fitness_fail": "notify_on_model_fitness_fail",
"quality_regression": "notify_on_quality_regression",
"audit_chain_break": "notify_on_audit_chain_break",
"tls_expiry": "notify_on_tls_expiry",
"oracle_paused": "notify_on_oracle_failure",
}.get(kind)
if flag is None:
return False
return bool(getattr(settings, flag, False))
async def _post_with_retries(url: str, payload: dict[str, Any], *, verify: bool) -> int:
"""POST *payload* to *url* with a short timeout + bounded transient retries.
Returns the final HTTP status code. Raises on a transport error that outlives
the retry budget (the caller swallows it). A 4xx is terminal (returned as-is);
a 5xx is retried up to :data:`_MAX_RETRIES`.
"""
import asyncio # noqa: PLC0415 - local; keeps module import cheap
attempt = 0
async with httpx.AsyncClient(timeout=_SEND_TIMEOUT_S, verify=verify) as client:
while True:
try:
resp = await client.post(url, json=payload)
except (httpx.ConnectError, httpx.ConnectTimeout):
# A connect failure is the transient class the retry budget
# exists for: the request never reached the receiver, and a
# momentary reset must not drop the page because the dedup
# slot for this hour is already taken. A read timeout or a
# torn response is not retried: the receiver may have taken
# the POST, and a page delivered twice is worse than one
# delivered slowly. Anything else (bad URL, bug) propagates.
if attempt >= _MAX_RETRIES:
raise
else:
if resp.status_code < 500 or attempt >= _MAX_RETRIES:
return resp.status_code
attempt += 1
await asyncio.sleep(_RETRY_BACKOFF_S * attempt)
async def fire(event: NotifyEvent, settings: Any, audit: Any = None) -> None:
"""Send *event* to the configured webhook — fail-soft, zero-egress when off.
THE ZERO-EGRESS GUARANTEE: this returns BEFORE constructing any ``httpx``
client unless ALL of these hold:
* ``settings.notify_enabled`` is True (the master switch), AND
* the per-trigger toggle for ``event.kind`` is on (``test`` bypasses this),
AND
* ``settings.notify_webhook_url`` is set.
When any is false, NO network I/O happens (no client is built) and the
function returns immediately. A duplicate within the same hour is also a
no-op (dedup), with no egress.
Every ATTEMPTED send (one that passes the gates + dedup) is audited under the
``"notification"`` kind, best-effort — an audit failure is logged and
swallowed. A webhook failure (timeout, 5xx after retries, transport error) is
logged and swallowed too: a webhook must NEVER break the investigation or hunt
that fired it.
"""
if not bool(getattr(settings, "notify_enabled", False)):
return # master switch off → zero egress
if not _trigger_enabled(settings, event.kind):
return # per-trigger toggle off (or unknown kind) → zero egress
url = _webhook_url(settings)
if not url:
return # no destination → zero egress
if not _dedup_should_send(event):
return # already pinged this (kind, entity) this hour → zero egress
fmt = str(getattr(settings, "notify_format", "json") or "json").lower()
if fmt not in NOTIFY_FORMATS:
fmt = "json"
payload = _build_payload(event, fmt)
verify = bool(getattr(settings, "notify_verify_ssl", True))
status: int | None = None
error: str | None = None
try:
# The demo guard sits INSIDE the try so a blocked send stays fail-soft
# (a webhook must never break the caller) while still refusing before
# any HTTP client is constructed.
assert_egress_allowed(settings, "webhook notify")
status = await _post_with_retries(url, payload, verify=verify)
except Exception as exc: # a webhook failure must never break the caller
error = type(exc).__name__
_LOGGER.warning("notification webhook send failed: %s", error)
# Audit EVERY attempted send (success or failure), best-effort. The URL is a
# secret — it is NEVER placed in the audit payload; only the format + outcome.
if audit is not None:
try:
await audit.log_kind(
session_id=f"notify:{event.kind}",
kind="notification",
payload={
"notify_kind": event.kind,
"format": fmt,
"severity": event.severity,
"ok": error is None and status is not None and status < 400,
"status": status,
"error": error,
},
)
except Exception: # audit is best-effort — never break a send on it
_LOGGER.warning("notification audit write failed (continuing)", exc_info=True)
async def fire_safe(event: NotifyEvent, settings: Any, audit: Any = None) -> None:
""":func:`fire`, wrapped so it can NEVER raise into the primary flow.
The trigger sites (investigation finalize, hunt finalize, model-fitness) call
THIS from inside the recorded-run/probe path. ``fire`` is already fail-soft on
webhook + audit errors, but this belt-and-suspenders wrapper also swallows a
misconfiguration bug (a bad settings double, a programming error building the
event) so a webhook can never break an investigation or hunt.
"""
try:
await fire(event, settings, audit)
except Exception: # a notification must NEVER break the primary flow
_LOGGER.warning("notify.fire_safe swallowed an unexpected error", exc_info=True)
def event_for_investigation(
*, investigation_id: str, report: dict[str, Any], settings: Any
) -> NotifyEvent | None:
"""Build a TP NotifyEvent from a finalized investigation report, or None.
Returns None (no notification) unless the verdict is ``true_positive`` AND the
confidence is at/above ``notify_tp_confidence_threshold`` AND ``notify_on_tp``
is on. Pure + side-effect-free — the caller decides whether to fire. The body
carries the verdict/confidence + a short summary; the permalink is
``/app/investigation/``.
"""
if not bool(getattr(settings, "notify_on_tp", False)):
return None
if str(report.get("verdict") or "") != "true_positive":
return None
try:
confidence = float(report.get("confidence") or 0.0)
except (TypeError, ValueError):
return None
threshold = float(getattr(settings, "notify_tp_confidence_threshold", 0.9))
if confidence < threshold:
return None
summary = str(report.get("summary") or "").strip()
body = f"Confidence {confidence:.2f}. {summary}".strip()
return NotifyEvent(
kind="tp",
title="True positive found",
body=body,
url=f"/app/investigation/{investigation_id}",
severity="critical",
)
def event_for_hunt(
*,
hunt_id: str,
report: dict[str, Any],
settings: Any,
verdicts: FindingVerdicts | None = None,
) -> NotifyEvent | None:
"""Build a hunt-threat NotifyEvent from a finalized hunt report, or None.
Returns None unless ``notify_on_hunt_threat`` is on AND the report has at least
one finding with ``category == 'threat'``. The body names the threat count +
the worst finding's title; the permalink is ``/app/hunts/``.
``verdicts`` is the summary of the investigations of the hunt's subject
findings. When one of them reached a verdict, the verdicts decide: the
notice fires on a true positive only, and its title states the count
("Hunt finished: 2 of 3 findings are threats").
"""
if not bool(getattr(settings, "notify_on_hunt_threat", False)):
return None
findings = report.get("findings")
if not isinstance(findings, list):
return None
if verdicts is not None and verdicts.decided:
if not verdicts.threats:
return None
label = outcome_label("threats_confirmed", verdicts)
return NotifyEvent(
kind="hunt_threat",
title=f"Hunt finished: {label}",
body=f"{label}. The investigations of the findings decided it."[:500],
url=f"/app/hunts/{hunt_id}",
severity="critical",
)
threats = [
f for f in findings if isinstance(f, dict) and str(f.get("category") or "") == "threat"
]
if not threats:
return None
lead = str(threats[0].get("title") or "").strip()
n = len(threats)
body = f"{n} threat finding{'s' if n != 1 else ''}. {lead}".strip()
return NotifyEvent(
kind="hunt_threat",
title="Hunt found a threat",
body=body,
url=f"/app/hunts/{hunt_id}",
severity="critical",
)
def event_for_model_fitness(*, result: dict[str, Any], settings: Any) -> NotifyEvent | None:
"""Build a model-fitness-FAIL NotifyEvent from a probe result, or None.
Returns None unless ``notify_on_model_fitness_fail`` is on AND the probe graded
``fail``. No permalink (config is not a per-entity resource) — the dedup key
then falls back to the title, which names the model, so repeated FAILs on the
same model dedup within the hour. ``detail`` is already scrubbed by probes.py.
"""
if not bool(getattr(settings, "notify_on_model_fitness_fail", False)):
return None
if str(result.get("grade") or "") != "fail":
return None
model = str(result.get("model") or "analyst model")
detail = str(result.get("detail") or "").strip()
return NotifyEvent(
kind="model_fitness_fail",
title=f"Analyst model unfit: {model}",
body=detail or "The model fitness probe graded this model FAIL.",
url="",
severity="warning",
)
def event_for_quality_regression(
*, mode: str, reasons: list[str], settings: Any
) -> NotifyEvent | None:
"""Build a quality-regression NotifyEvent from a nightly's alarm, or None.
Returns None unless ``notify_on_quality_regression`` is on AND the detector
produced at least one reason. The body carries the measurement mode (so
on-call knows whether "agreement" even applies) plus the detector's own
reason strings — they already embed the numbers. Permalink is the dashboard
(the Quality card lives there); it doubles as the dedup entity, so repeated
nightly alarms collapse to one ping per hour.
"""
if not bool(getattr(settings, "notify_on_quality_regression", False)):
return None
if not reasons:
return None
mode_label = (
"The Oracle graded this run." if mode == "graded" else "soc-ai measured this run locally."
)
body = f"The nightly micro-eval found a regression. {mode_label} " + ". ".join(reasons)
return NotifyEvent(
kind="quality_regression",
title="Verdict quality regression",
body=body,
url="/app/",
severity="warning",
)
def event_for_audit_chain_break(
*,
epochs_broken: int,
seq: int | None,
kind: str | None,
detail: str | None,
latest_epoch_broken: bool,
settings: Any,
blast_radius: str | None = None,
) -> NotifyEvent | None:
"""Build an audit-chain-break NotifyEvent, or None when the trigger is off.
Critical severity, and no apology for it: the audit trail is the record
every other claim this product makes rests on, and a break in it means
either a concurrency defect or someone editing decisions after the fact.
The body carries WHICH of those the verifier could tell (see
:data:`~soc_ai.audit.chain.BreakKind`) so on-call is not left to guess
between a known bug and an intrusion. The permalink is the config page,
where the verify-chain diagnostic lives; it doubles as the dedup entity,
so a standing break pings once an hour at worst.
``blast_radius`` says how widespread the damage is (see
:func:`~soc_ai.audit.verify.describe_blast_radius`). One sequence number
out of 41 is not something on-call can size, and the difference between one
collision and a forked afternoon changes what they do next.
"""
if not bool(getattr(settings, "notify_on_audit_chain_break", False)):
return None
where = f" at sequence {seq}" if seq is not None else ""
if latest_epoch_broken:
scope = "the current epoch"
elif epochs_broken == 1:
scope = "1 epoch of history"
else:
scope = f"{epochs_broken} epochs of history"
body = detail or f"The audit hash chain does not verify{where}."
if blast_radius:
body = f"{body} {blast_radius}"
return NotifyEvent(
kind="audit_chain_break",
title=f"Audit chain broken: {kind or 'unknown'} in {scope}",
body=body,
url="/app/config",
severity="critical",
)
def event_for_tls_expiry(
*, subject: str, days_left: int, band: int, settings: Any
) -> NotifyEvent | None:
"""The certificate soc-ai serves with is close to expiry, or expired.
Critical at 7 days and below, and once expired. Warning at 30 and 14 days.
The permalink is the TLS panel on the config page. It doubles as the dedup
entity, so the band sits in the URL and the hourly dedup keys per band.
The daily loop fires once per band change, so on-call hears about each
band once.
"""
if not bool(getattr(settings, "notify_on_tls_expiry", False)):
return None
if band == 0:
title = f"TLS certificate expired: {subject}"
body = (
"The certificate soc-ai serves with has expired. Browsers refuse it. "
"Install a new certificate and restart soc-ai."
)
severity = "critical"
else:
title = f"TLS certificate expires in {days_left} days: {subject}"
body = (
f"The certificate soc-ai serves with expires in {days_left} days. "
"Install a new certificate and restart soc-ai."
)
severity = "critical" if band <= 7 else "warning"
return NotifyEvent(
kind="tls_expiry",
title=title,
body=body,
url=f"/app/config?band={band}#tls",
severity=severity,
)
def oracle_pause_text(*, pause_reason: str, until: str | None) -> tuple[str, str]:
"""The title and the body of an Oracle pause, for the bell and the webhook."""
when = f"{until.replace('T', ' ').replace('Z', '')} UTC" if until else "the reset time"
title = f"Oracle calls paused until {when}"
if pause_reason == "quota":
body = (
"The Oracle route answered with a usage limit. soc-ai makes no Oracle call "
f"until {when}. Each escalation in the pause keeps its local verdict and is "
"recorded as oracle_skipped."
)
else:
body = (
"The Oracle route answered with three server errors in a row. soc-ai makes "
f"no Oracle call until {when}. Each escalation in the pause keeps its local "
"verdict and is recorded as oracle_skipped."
)
return title, body
def event_for_oracle_paused(
*, pause_reason: str, until: str | None, settings: Any
) -> NotifyEvent | None:
"""The Oracle route paused, or None when the trigger is off.
Fired once, when the pause opens (soc_ai.oracle.breaker returns the
transition). The permalink is the Oracle section of the config page. The
pause start rides in the URL, so the hourly dedup keys per pause.
"""
if not bool(getattr(settings, "notify_on_oracle_failure", False)):
return None
title, body = oracle_pause_text(pause_reason=pause_reason, until=until)
return NotifyEvent(
kind="oracle_paused",
title=title,
body=body,
url=f"/app/config?oracle_paused={until or ''}#oracle",
severity="warning",
)
def canned_test_event() -> NotifyEvent:
"""The synthetic event the Test button sends.
Contains NO internal identifier — a fixed, unmistakably-synthetic body — so an
operator validating the webhook before enabling notifications never leaks a
real alert/hunt/host into the destination channel.
"""
return NotifyEvent(
kind="test",
title="soc-ai notification test",
body=("This is a test notification from soc-ai. If you read it, the webhook works."),
url="",
severity="info",
)
def webhook_configured(settings: Any) -> bool:
"""True iff a non-empty webhook URL is configured. No egress."""
return bool(_webhook_url(settings))
async def send_test(settings: Any, audit: Any = None) -> tuple[bool, str]:
"""Send the canned test event NOW, bypassing the master toggle + dedup.
The Test button is an explicit operator validation: it sends regardless of
``notify_enabled`` (so the operator can prove the destination works BEFORE
switching routing on) and regardless of dedup, but it STILL requires a
configured webhook URL. The audited outcome uses the same ``"notification"``
kind as a real send. Returns ``(ok, detail)`` where ``detail`` is a short,
secret-free human string — it NEVER contains the webhook URL.
Never raises: a transport/timeout error becomes ``(False, "")``.
"""
url = _webhook_url(settings)
if not url:
return False, "No webhook URL is configured."
fmt = str(getattr(settings, "notify_format", "json") or "json").lower()
if fmt not in NOTIFY_FORMATS:
fmt = "json"
event = canned_test_event()
payload = _build_payload(event, fmt)
verify = bool(getattr(settings, "notify_verify_ssl", True))
status: int | None = None
error: str | None = None
try:
# Demo guard inside the try: the Test button reports a structured
# failure instead of raising, and no HTTP client is ever constructed.
assert_egress_allowed(settings, "webhook notify")
status = await _post_with_retries(url, payload, verify=verify)
except Exception as exc: # never raise into the route
error = type(exc).__name__
_LOGGER.warning("notification test send failed: %s", error)
ok = error is None and status is not None and status < 400
if audit is not None:
try:
await audit.log_kind(
session_id="notify:test",
kind="notification",
payload={
"notify_kind": "test",
"format": fmt,
"ok": ok,
"status": status,
"error": error,
},
)
except Exception: # audit is best-effort
_LOGGER.warning("notification test audit write failed (continuing)", exc_info=True)
if ok:
return True, f"soc-ai sent the test. The webhook returned HTTP {status}."
if error is not None:
return False, f"The test send failed: {error}."
return False, f"The webhook returned HTTP {status}."