from __future__ import annotations import asyncio import concurrent.futures import base64 import contextlib import hashlib import hmac import json import logging import math import os import re import secrets import settings import time from collections import defaultdict, deque from datetime import datetime, timezone from pathlib import Path from threading import Lock, RLock from typing import Any, Dict from urllib.parse import quote import httpx from fastapi import FastAPI, Request from fastapi.responses import HTMLResponse, JSONResponse, Response, StreamingResponse from starlette.background import BackgroundTask import control_plane_client import host_store import internal_api from env_secrets import load_env_secrets from policy_templates import template_catalog from shim import _CACHE_READ_FACTOR # the billing cache-read discount — one source # Operator-managed keys/consumer-hashes live on the PVC (.env.secrets) and are # the source of truth — load them over the container env BEFORE reading any of # the values below, so dashboard edits survive pod restarts. load_env_secrets() UPSTREAM = os.getenv("ROUTER_UPSTREAM", "http://router:18080").rstrip("/") def _router_admin_headers(extra: dict[str, str] | None = None) -> dict[str, str]: """Headers for the router's operator /x/* endpoints (wallet, providers, provider-key, reloads, sessions/calls), which require the shared internal secret. Only ever sent on these server-originated calls, never on proxied client traffic.""" headers = dict(extra or {}) if control_plane_client.CONTROL_PLANE_INTERNAL_SECRET: headers["x-internal-secret"] = control_plane_client.CONTROL_PLANE_INTERNAL_SECRET return headers CALLER_KEYS_JSON = os.getenv("CALLER_KEYS_JSON", "{}") CALLER_KEYS_SHA256_JSON = os.getenv("CALLER_KEYS_SHA256_JSON", "{}") # GitOps/bootstrap keys arrive from a reconciled workload Secret. Keep them in # a separate channel because load_env_secrets() intentionally lets the # dashboard-managed PVC override CALLER_KEYS_JSON; replacing the whole map # would otherwise make a reconciled workload key disappear after the first # dashboard-issued key is persisted. CALLER_KEYS_BOOTSTRAP_JSON = os.getenv("CALLER_KEYS_BOOTSTRAP_JSON", "{}") RATE_PER_MIN = int(os.getenv("RATE_PER_MIN", "600")) BURST = int(os.getenv("BURST", "200")) # Per-replica overload guard. Zero keeps the legacy unlimited behaviour for # local/single-pod installs; Kubernetes sets explicit bounds and autoscales from # the exported active+pending gauges. MAX_INFLIGHT_REQUESTS = max(0, int(os.getenv("MAX_INFLIGHT_REQUESTS", "0"))) MAX_PENDING_REQUESTS = max(0, int(os.getenv("MAX_PENDING_REQUESTS", "0"))) CAPACITY_QUEUE_TIMEOUT_S = max(0.0, float(os.getenv("CAPACITY_QUEUE_TIMEOUT_S", "2"))) def _reservation_usd() -> float: raw = os.getenv("CLOUD_BUDGET_RESERVATION_USD", "0.05") try: value = float(raw) except (TypeError, ValueError): return 0.05 return value if math.isfinite(value) and value > 0 else 0.05 # Held against a control-plane key subject's monthly budget per in-flight call # and replaced by the actual cost when the call is recorded; overshoot is # bounded by concurrent calls x (actual cost - reservation). CLOUD_BUDGET_RESERVATION_USD = _reservation_usd() KEY_BUDGET_RESERVATION_TTL_S = 900.0 UPSTREAM_MAX_CONNECTIONS = max(1, int(os.getenv("UPSTREAM_MAX_CONNECTIONS", "100"))) UPSTREAM_MAX_KEEPALIVE_CONNECTIONS = max( 1, int(os.getenv("UPSTREAM_MAX_KEEPALIVE_CONNECTIONS", "40"))) RECENT_LIMIT = int(os.getenv("DASHBOARD_RECENT_LIMIT", "200")) DASHBOARD_STATS_RECENT_LIMIT = max(1, min(int(os.getenv("DASHBOARD_STATS_RECENT_LIMIT", "100")), 100)) DASHBOARD_TRUSTED_USER_HEADER = os.getenv("DASHBOARD_TRUSTED_USER_HEADER", "").strip() DASHBOARD_TRUSTED_USER_SECRET = os.getenv("DASHBOARD_TRUSTED_USER_SECRET", "") DASHBOARD_PASSWORD_SHA256 = os.getenv("DASHBOARD_PASSWORD_SHA256", "") DASHBOARD_SESSION_SECRET = os.getenv("DASHBOARD_SESSION_SECRET", "") # Local-dev escape hatch: when truthy, the dashboard skips auth entirely and every # request is treated as a local admin. OFF by default — never enable in a # deployment reachable by anyone but you. DASHBOARD_NO_AUTH = os.getenv("DASHBOARD_NO_AUTH", "").strip().lower() in ("1", "true", "yes", "on") DASHBOARD_COOKIE_NAME = os.getenv("DASHBOARD_COOKIE_NAME", "router_dashboard_session") DASHBOARD_COOKIE_MAX_AGE = int(os.getenv("DASHBOARD_COOKIE_MAX_AGE", "2592000")) DASHBOARD_COOKIE_PATH = os.getenv("DASHBOARD_COOKIE_PATH", "/dashboard") DASHBOARD_KEY_ENV_PATH = os.getenv("DASHBOARD_KEY_ENV_PATH", "/run/llm-router/.env.secrets") CODEX_ACCOUNTS_DIR = os.getenv("CODEX_ACCOUNTS_DIR", "/codex/accounts") CODEX_AUTH_PATH = os.getenv("CODEX_AUTH_PATH") or None CODEX_BROKER_URL = os.getenv("CODEX_BROKER_URL", "").rstrip("/") CODEX_BROKER_TOKEN = os.getenv("CODEX_BROKER_TOKEN", "") DASHBOARD_KEY_PREFIX = os.getenv("DASHBOARD_KEY_PREFIX", "llmr") DEFAULT_ROTATION_GRACE_S = int(os.getenv("DASHBOARD_KEY_ROTATION_GRACE_S", "86400")) DASHBOARD_POLICY_CONFIG_PATH = os.getenv("DASHBOARD_POLICY_CONFIG_PATH", "config.live.lua") DASHBOARD_POLICY_METRICS_PATH = os.getenv("DASHBOARD_POLICY_METRICS_PATH", "metrics.live.lua") DASHBOARD_POLICY_DIR = os.getenv("DASHBOARD_POLICY_DIR", "policies") ROUTER_CONTEXT_LENGTH = int(os.getenv("ROUTER_CONTEXT_LENGTH", "200000")) ROUTE_HEALTH_ROUTES = [r.strip() for r in os.getenv("DASHBOARD_ROUTE_HEALTH_ROUTES", "profile:default").split(",") if r.strip()] SYNTHETIC_PROBES_ENABLED = os.getenv("DASHBOARD_SYNTHETIC_PROBES_ENABLED", "1").lower() not in {"0", "false", "no", "off"} RUN_COST_BACKFILL = os.getenv("RUN_COST_BACKFILL", "1").lower() not in { "0", "false", "no", "off"} SYNTHETIC_PROBE_INTERVAL_S = float(os.getenv("DASHBOARD_SYNTHETIC_PROBE_INTERVAL_S", "300")) SYNTHETIC_PROBE_INITIAL_DELAY_S = float(os.getenv("DASHBOARD_SYNTHETIC_PROBE_INITIAL_DELAY_S", "45")) SYNTHETIC_PROBE_TIMEOUT_S = float(os.getenv("DASHBOARD_SYNTHETIC_PROBE_TIMEOUT_S", "45")) SYNTHETIC_PROBE_CALLER = os.getenv("DASHBOARD_SYNTHETIC_PROBE_CALLER", "dashboard-probe") # Optional namespace for control-plane-resolved callers (e.g. "cp:") so tenant # slugs can't merge attribution with operator-minted consumer names. Default off: # calls.caller == tenant slug, which is what the control plane's usage reads key on. CONTROL_PLANE_CALLER_PREFIX = os.getenv("CONTROL_PLANE_CALLER_PREFIX", "") def _load_caller_map(raw: str, name: str) -> Dict[str, str]: try: data = json.loads(raw) if not isinstance(data, dict): raise ValueError(f"{name} must be a JSON object") if not all(isinstance(k, str) and isinstance(v, str) for k, v in data.items()): raise ValueError(f"{name} keys and values must be strings") return data except Exception as exc: raise RuntimeError(f"invalid {name}: {exc}") from exc def _bootstrap_caller_key_hashes(raw: str) -> Dict[str, str]: """Load GitOps keys as hashes so the dashboard can never reveal them.""" return { hashlib.sha256(token.encode()).hexdigest(): owner for token, owner in _load_caller_map( raw, "CALLER_KEYS_BOOTSTRAP_JSON").items() } CALLER_KEYS: Dict[str, str] = _load_caller_map( CALLER_KEYS_JSON, "CALLER_KEYS_JSON") CALLER_KEY_HASHES: Dict[str, str] = _bootstrap_caller_key_hashes( CALLER_KEYS_BOOTSTRAP_JSON) CALLER_KEY_HASHES.update(_load_caller_map( CALLER_KEYS_SHA256_JSON, "CALLER_KEYS_SHA256_JSON")) logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(message)s") log = logging.getLogger("llm-router-auth-proxy") app = FastAPI(title="llm-router auth proxy", docs_url=None, redoc_url=None) # Control-plane metering surface (/internal/usage[/recent]). Registered before # any route definition below so Starlette matches it ahead of the catch-all # proxy; hidden (404) unless CONTROL_PLANE_INTERNAL_SECRET is configured. app.include_router(internal_api.router) _client: httpx.AsyncClient | None = None _probe_task: asyncio.Task[None] | None = None _started_wall = time.time() _stats_lock = RLock() # Event-loop-owned capacity state. No thread lock is needed: every mutation is # made by an auth-proxy coroutine, and the synchronous metrics renderer only # snapshots it on that same loop. _capacity: asyncio.Semaphore | None = None _active_requests = 0 _pending_requests = 0 # Low-cardinality Prometheus state. Caller/model/provider labels are # deliberately absent: the durable ledger owns those dimensions and putting # them here would make autoscaling telemetry attacker-controlled. _metrics_lock = Lock() _metric_requests: dict[str, int] = defaultdict(int) _metric_rejects: dict[str, int] = defaultdict(int) _metric_store_errors: dict[str, int] = defaultdict(int) _duration_buckets = (0.1, 0.5, 1.0, 2.5, 5.0, 10.0, 20.0, 30.0, 50.0, 60.0, 120.0) _metric_duration_bucket: dict[float, int] = defaultdict(int) _metric_duration_count = 0 _metric_duration_sum = 0.0 def _counter() -> dict[str, Any]: return { "requests": 0, "errors": 0, "tokens_in": 0, "tokens_out": 0, "tokens_total": 0, "tokens_cached": 0, "latency_ms_total": 0.0, "latency_ms_max": 0.0, "last_seen": None, } def _new_stats() -> dict[str, Any]: return { "total_requests": 0, "total_rejects": 0, "total_errors": 0, "total_tokens_in": 0, "total_tokens_out": 0, "total_tokens": 0, "by_caller": defaultdict(_counter), "by_provider": defaultdict(_counter), "by_model_family": defaultdict(_counter), "by_route": defaultdict(_counter), "by_served_model": defaultdict(_counter), "by_status": defaultdict(int), "by_caller_provider": defaultdict(lambda: defaultdict(_counter)), "by_caller_model_family": defaultdict(lambda: defaultdict(_counter)), "by_caller_route": defaultdict(lambda: defaultdict(_counter)), "by_caller_served_model": defaultdict(lambda: defaultdict(_counter)), "by_caller_status": defaultdict(lambda: defaultdict(int)), "by_key_sha256": defaultdict(_counter), "by_key_provider": defaultdict(lambda: defaultdict(_counter)), "by_key_model_family": defaultdict(lambda: defaultdict(_counter)), "by_key_route": defaultdict(lambda: defaultdict(_counter)), "by_key_served_model": defaultdict(lambda: defaultdict(_counter)), "by_key_status": defaultdict(lambda: defaultdict(int)), "key_owner": {}, "recent": deque(maxlen=RECENT_LIMIT), "synthetic_route_health": {}, } async def _capacity_acquire() -> bool: """Enter the bounded upstream section, queueing only a small burst. HPA/KEDA cannot create a pod before the first burst arrives. The short bounded queue absorbs that reaction window; once both active and pending budgets are full we shed immediately instead of letting arbitrary numbers of 50-second requests consume sockets and kill the health loop. """ global _active_requests, _pending_requests if _capacity is None: _active_requests += 1 return True queued = _capacity.locked() if queued: if _pending_requests >= MAX_PENDING_REQUESTS: return False _pending_requests += 1 try: if queued: await asyncio.wait_for( _capacity.acquire(), timeout=CAPACITY_QUEUE_TIMEOUT_S) else: await _capacity.acquire() except (asyncio.TimeoutError, TimeoutError): return False finally: if queued: _pending_requests = max(0, _pending_requests - 1) _active_requests += 1 return True def _capacity_release() -> None: global _active_requests _active_requests = max(0, _active_requests - 1) if _capacity is not None: _capacity.release() def _metric_request(status: int, latency_ms: float) -> None: global _metric_duration_count, _metric_duration_sum seconds = max(0.0, float(latency_ms) / 1000.0) with _metrics_lock: _metric_requests[str(int(status))] += 1 _metric_duration_count += 1 _metric_duration_sum += seconds for bound in _duration_buckets: if seconds <= bound: _metric_duration_bucket[bound] += 1 def _metric_reject(reason: str) -> None: with _metrics_lock: _metric_rejects[str(reason or "unknown")] += 1 def _metric_store_error(kind: str) -> None: with _metrics_lock: _metric_store_errors[str(kind or "unknown")] += 1 def _prometheus_metrics() -> str: """Render the small autoscaling/availability surface without a dependency. The endpoint is scraped directly on the pod's auth-proxy port. nginx blocks it on the public listener in the Kubernetes manifest. """ with _metrics_lock: requests = dict(_metric_requests) rejects = dict(_metric_rejects) store_errors = dict(_metric_store_errors) duration_buckets = dict(_metric_duration_bucket) duration_count = _metric_duration_count duration_sum = _metric_duration_sum lines = [ "# HELP llm_router_inflight_requests Requests currently executing upstream.", "# TYPE llm_router_inflight_requests gauge", f"llm_router_inflight_requests {_active_requests}", "# HELP llm_router_pending_requests Requests waiting for per-pod capacity.", "# TYPE llm_router_pending_requests gauge", f"llm_router_pending_requests {_pending_requests}", "# HELP llm_router_capacity_requests Configured per-pod active request capacity.", "# TYPE llm_router_capacity_requests gauge", f"llm_router_capacity_requests {MAX_INFLIGHT_REQUESTS}", "# HELP llm_router_requests_total Completed authenticated upstream requests.", "# TYPE llm_router_requests_total counter", ] for status, count in sorted(requests.items()): lines.append(f'llm_router_requests_total{{status="{status}"}} {count}') lines.extend([ "# HELP llm_router_rejections_total Requests rejected before upstream.", "# TYPE llm_router_rejections_total counter", ]) for reason, count in sorted(rejects.items()): safe = re.sub(r"[^A-Za-z0-9_.:-]", "_", reason) lines.append(f'llm_router_rejections_total{{reason="{safe}"}} {count}') lines.extend([ "# HELP llm_router_store_errors_total Admission-store failures.", "# TYPE llm_router_store_errors_total counter", ]) for kind, count in sorted(store_errors.items()): safe = re.sub(r"[^A-Za-z0-9_.:-]", "_", kind) lines.append(f'llm_router_store_errors_total{{kind="{safe}"}} {count}') lines.extend([ "# HELP llm_router_request_duration_seconds End-to-end upstream request latency.", "# TYPE llm_router_request_duration_seconds histogram", ]) for bound in _duration_buckets: lines.append( f'llm_router_request_duration_seconds_bucket{{le="{bound:g}"}} ' f'{duration_buckets.get(bound, 0)}') lines.extend([ f'llm_router_request_duration_seconds_bucket{{le="+Inf"}} {duration_count}', f"llm_router_request_duration_seconds_sum {duration_sum:.6f}", f"llm_router_request_duration_seconds_count {duration_count}", "# HELP llm_router_process_uptime_seconds Auth-proxy process uptime.", "# TYPE llm_router_process_uptime_seconds gauge", f"llm_router_process_uptime_seconds {max(0.0, time.time() - _started_wall):.3f}", "", ]) return "\n".join(lines) _stats: dict[str, Any] = _new_stats() # Short TTL cache over the SQL-derived dashboard snapshots (stats bundle, the # logins/provider-keys rollups) so the dashboard's 15s auto-refresh and multiple # concurrent viewers don't re-run the window aggregation per request. Keyed by # the full query tuple; invalidated by TTL only (freshness lag <= the TTL, less # than the frontend's own refresh period). Never used for timeframe="runtime" # (that path is in-memory and already O(1)). _SNAPSHOT_TTL_S = float(os.getenv("DASHBOARD_STATS_TTL_S", "12")) _snapshot_cache: dict[tuple, tuple[float, Any]] = {} _snapshot_cache_lock = RLock() _snapshot_compute_locks: dict[tuple, Lock] = {} def _snapshot_cache_get(key: tuple) -> Any: if _SNAPSHOT_TTL_S <= 0: return None with _snapshot_cache_lock: hit = _snapshot_cache.get(key) if hit and (time.monotonic() - hit[0]) < _SNAPSHOT_TTL_S: return hit[1] return None def _snapshot_cache_put(key: tuple, value: Any) -> None: if _SNAPSHOT_TTL_S <= 0: return with _snapshot_cache_lock: if len(_snapshot_cache) > 256: # bound the key space (bad params etc.) _snapshot_cache.clear() _snapshot_cache[key] = (time.monotonic(), value) def _snapshot_cached_compute(key: tuple, compute: Any) -> Any: """Compute a missing snapshot once, even with concurrent dashboard loads.""" value = _snapshot_cache_get(key) if value is not None: return value with _snapshot_cache_lock: lock = _snapshot_compute_locks.setdefault(key, Lock()) with lock: value = _snapshot_cache_get(key) if value is None: value = compute() _snapshot_cache_put(key, value) return value def _snapshot_cache_clear() -> None: with _snapshot_cache_lock: _snapshot_cache.clear() def _reset_stats_for_tests() -> None: """Reset in-memory counters. Drains the host-store write queue first so a just-recorded call has landed in the `calls` ledger before the test reads the derived stats back (the persistent store now backs them, #5). Also drops the snapshot TTL cache so the next read reflects the reset.""" host_store._write_q.join() _snapshot_cache_clear() with _stats_lock: _stats.clear() _stats.update(_new_stats()) def _consumers() -> list[str]: return sorted(set(CALLER_KEYS.values()) | set(CALLER_KEY_HASHES.values()) | set(_load_issued_keys().keys())) def _safe_consumer_name(raw: str) -> str: name = re.sub(r"[^A-Za-z0-9_.:-]+", "-", (raw or "").strip()).strip("-._:") if not name or len(name) > 80: raise ValueError("consumer must be 1-80 chars: letters, numbers, dot, underscore, colon, or dash") return name def _write_json_file(path: Path, data: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(path.suffix + ".tmp") tmp.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n") tmp.chmod(0o600) tmp.replace(path) path.chmod(0o600) def _provider_env_error(key: str, value: str) -> str | None: """Why KEY=value may not be written to the env-secrets file (applied over the process env at boot), or None. Only provider-credential names, never an infrastructure/security variable, and no control characters: a newline in the value would inject arbitrary extra KEY=value lines.""" from provider_overlay import AUTH_ENV_RE, is_forbidden_auth_env if not AUTH_ENV_RE.match(key or "") or is_forbidden_auth_env(key): return f"auth_env {key!r} is not an allowed provider credential name" if any(ord(ch) < 0x20 or ord(ch) == 0x7F for ch in f"{key}{value}"): return "key must not contain control characters" return None def _upsert_env_line(path: Path, key: str, value: str) -> None: """Set KEY=value in an env file (used for provider API keys). Same semantics as _upsert_env_json but for plain string values.""" if (error := _provider_env_error(key, value)) is not None: raise ValueError(error) path.parent.mkdir(parents=True, exist_ok=True) lines = path.read_text().splitlines() if path.exists() else [] rendered = f"{key}={value}" out: list[str] = [] replaced = False for line in lines: if line.startswith(key + "="): out.append(rendered) replaced = True else: out.append(line) if not replaced: out.append(rendered) path.write_text("\n".join(out) + "\n") path.chmod(0o600) def _upsert_env_json(path: Path, key: str, value: dict[str, str]) -> None: path.parent.mkdir(parents=True, exist_ok=True) lines = path.read_text().splitlines() if path.exists() else [] rendered = key + "=" + json.dumps(value, sort_keys=True, separators=(",", ":")) out: list[str] = [] replaced = False for line in lines: if line.startswith(key + "="): out.append(rendered) replaced = True else: out.append(line) if not replaced: out.append(rendered) path.write_text("\n".join(out) + "\n") path.chmod(0o600) _issued_keys_load_failed = False def _load_issued_keys() -> dict[str, Any]: global _issued_keys_load_failed records, ok = host_store.get_consumer_keys() _issued_keys_load_failed = not ok return records def _clean_route_list(value: Any) -> list[str]: if value is None or value == "": return [] if isinstance(value, str): items = re.split(r"[\s,]+", value) elif isinstance(value, list): items = [str(v) for v in value] else: return [] out = [] for item in items: route = item.strip() if route and len(route) <= 160 and route not in out: out.append(route) return out def _optional_int(value: Any, *, min_value: int = 0, max_value: int | None = 1_000_000) -> int | None: if value is None or value == "": return None try: ivalue = int(value) except (TypeError, ValueError): return None if max_value is not None: ivalue = min(max_value, ivalue) return max(min_value, ivalue) def _optional_float(value: Any, *, min_value: float = 0.0, max_value: float = 1_000_000.0) -> float | None: if value is None or value == "": return None try: number = float(value) except (TypeError, ValueError): return None if not (min_value <= number <= max_value): return None return round(number, 6) def _normalize_key_record(record: Any) -> dict[str, Any] | None: if not isinstance(record, dict): return None prefix = str(record.get("sha256_prefix") or "").strip().lower() if not re.fullmatch(r"[a-f0-9]{8,64}", prefix): return None status = str(record.get("status") or "active").strip().lower() if status not in {"active", "revoked"}: status = "revoked" out = { "sha256_prefix": prefix, "status": status, "created_at": _optional_int(record.get("created_at"), max_value=None), "viewer": str(record.get("viewer") or "")[:80] or None, "expires_at": _optional_int(record.get("expires_at"), max_value=None), "revoked_at": _optional_int(record.get("revoked_at"), max_value=None), "replaced_at": _optional_int(record.get("replaced_at"), max_value=None), } return {k: v for k, v in out.items() if v is not None} def _normalize_consumer_record(consumer: str, record: Any) -> dict[str, Any]: now = int(time.time()) if record is None: record = {} elif not isinstance(record, dict): return {"status": "inactive", "allowed_routes": [], "rate_per_min": None, "burst": None, "keys": [], "updated_at": now} status = str(record.get("status") or "active").strip().lower() if status not in {"active", "inactive"}: status = "inactive" keys = [] if "keys" in record and not isinstance(record.get("keys"), list): return {"status": "inactive", "allowed_routes": [], "rate_per_min": None, "burst": None, "keys": [], "updated_at": now} raw_keys = record.get("keys") or [] for item in raw_keys: normalized = _normalize_key_record(item) if not normalized: return {"status": "inactive", "allowed_routes": [], "rate_per_min": None, "burst": None, "keys": [], "updated_at": now} keys.append(normalized) # Backward compatibility with the older {consumer: {sha256_prefix, created_at, viewer}} shape. legacy = _normalize_key_record(record) if legacy and not any(k["sha256_prefix"] == legacy["sha256_prefix"] for k in keys): keys.append(legacy) for token, owner in CALLER_KEYS.items(): if owner == consumer: digest = hashlib.sha256(token.encode()).hexdigest() if not any(digest.startswith(k["sha256_prefix"]) for k in keys): keys.append({"sha256_prefix": digest[:12], "status": "active", "storage": "CALLER_KEYS_JSON"}) for digest, owner in CALLER_KEY_HASHES.items(): if owner == consumer and not any(digest.startswith(k["sha256_prefix"]) for k in keys): keys.append({"sha256_prefix": digest[:12], "status": "active", "storage": "CALLER_KEYS_SHA256_JSON"}) rate_per_min = _optional_int(record.get("rate_per_min"), min_value=1) burst = _optional_int(record.get("burst"), min_value=1) return { "status": status, "allowed_routes": _clean_route_list(record.get("allowed_routes")), "rate_per_min": rate_per_min, "burst": burst, "batch": str(record.get("batch") or "").strip()[:80] or None, "member_id": str(record.get("member_id") or "").strip()[:16] or None, "budget_usd": _optional_float(record.get("budget_usd"), min_value=0.01), "keys": keys, "updated_at": _optional_int(record.get("updated_at")) or now, } def _issued_consumer_records() -> dict[str, dict[str, Any]]: issued = _load_issued_keys() consumers = sorted(set(CALLER_KEYS.values()) | set(CALLER_KEY_HASHES.values()) | set(issued.keys())) return {name: _normalize_consumer_record(name, issued.get(name)) for name in consumers} def _write_issued_consumer_records( records: dict[str, dict[str, Any]], *, key_digests: dict[str, str] | None = None, delete_key_digests: list[tuple[str, str]] | None = None) -> bool: """Persist the consumer records; returns True on success, False on a persistence failure — a swallowed failure would let a key rotation/revocation be reported as saved while still working after a restart (a security hole).""" compact = {} for consumer, record in sorted(records.items()): normalized = _normalize_consumer_record(consumer, record) if normalized["status"] == "active" and not normalized["allowed_routes"] and normalized["rate_per_min"] is None and normalized["burst"] is None and not normalized["keys"]: continue compact[consumer] = normalized return host_store.set_consumer_keys( compact, key_digests=key_digests, delete_key_digests=delete_key_digests) def _exact_consumer_key_digests( *, hashes: dict[str, str] | None = None, plaintext: dict[str, str] | None = None) -> dict[str, str]: """Build the complete shared resolver map without retaining plaintext.""" out = dict(CALLER_KEY_HASHES if hashes is None else hashes) for token, owner in (CALLER_KEYS if plaintext is None else plaintext).items(): out[hashlib.sha256(token.encode()).hexdigest()] = owner return out def _publish_local_consumer_map(name: str, mapping: dict[str, str], target: dict[str, str]) -> None: """Refresh this process and best-effort PVC mirror after a DB commit. PostgreSQL is the cross-replica authority. A PVC write failure must not hide an already-issued credential from the operator; the database copy is sufficient for current and future stateless replicas. """ target.clear() target.update(mapping) try: _upsert_env_json(Path(DASHBOARD_KEY_ENV_PATH), name, mapping) except Exception as exc: # noqa: BLE001 _metric_store_error("consumer_key_pvc_mirror") log.warning("consumer key PVC mirror failed for %s: %s", name, exc) def _consumer_meta(consumer: str) -> dict[str, Any]: records = _issued_consumer_records() if _issued_keys_load_failed: meta = _normalize_consumer_record(consumer, {}) meta["status"] = "inactive" return meta return records.get(consumer, _normalize_consumer_record(consumer, {})) def _consumer_store_unavailable() -> JSONResponse: return JSONResponse(status_code=503, content={"error": { "message": "consumer key store is temporarily unavailable", "type": "server_error", "code": "consumer_store_unavailable"}}) def _writable_consumer_records() -> dict[str, dict[str, Any]] | None: """Records for a read-modify-write, or None when the store read failed: the fallback view holds only env-configured consumers, and writing it back would wipe every issued record (keys, budgets, revocations).""" records = _issued_consumer_records() return None if _issued_keys_load_failed else records async def _consumer_view_unavailable(role: str) -> JSONResponse | None: # A consumer dashboard view is scoped to that consumer's records; if the # store cannot confirm them, fail closed rather than render unscoped data. if role == "admin": return None _, ok = await asyncio.to_thread(host_store.get_consumer_keys) return None if ok else _consumer_store_unavailable() def _active_key_rows(keys: list[dict[str, Any]]) -> list[dict[str, Any]]: now = int(time.time()) rows = [] for key in keys: row = dict(key) if row.get("status") == "active" and row.get("expires_at") and int(row["expires_at"]) <= now: row["status"] = "expired" rows.append(row) return rows def _consumer_key_rows(by_caller_stats: dict[str, dict[str, Any]] | None = None) -> list[dict[str, Any]]: records = _issued_consumer_records() plaintext_counts: dict[str, int] = defaultdict(int) hash_counts: dict[str, int] = defaultdict(int) for consumer in CALLER_KEYS.values(): plaintext_counts[consumer] += 1 for consumer in CALLER_KEY_HASHES.values(): hash_counts[consumer] += 1 rows = [] for name in _consumers(): meta = records.get(name, _normalize_consumer_record(name, {})) rows.append({ "consumer": name, "configured": name in set(CALLER_KEYS.values()) or name in set(CALLER_KEY_HASHES.values()), "status": meta.get("status", "active"), "allowed_routes": meta.get("allowed_routes") or [], "rate_per_min": meta.get("rate_per_min"), "burst": meta.get("burst"), "batch": meta.get("batch"), "member_id": meta.get("member_id"), "budget_usd": meta.get("budget_usd"), "effective_rate_per_min": meta.get("rate_per_min") or RATE_PER_MIN, "effective_burst": meta.get("burst") or BURST, "issued_metadata": name in records, "stored_raw_key": plaintext_counts.get(name, 0) > 0, "plaintext_key_count": plaintext_counts.get(name, 0), "hash_key_count": hash_counts.get(name, 0), "keys": _active_key_rows(meta.get("keys") or []), "stats": (by_caller_stats or {}).get(name) or _counter_snapshot(_stats["by_caller"].get(name, _counter())), }) return rows def _plaintext_key_rows_for_consumer(consumer: str) -> list[dict[str, Any]]: rows = [] for token, owner in sorted(CALLER_KEYS.items(), key=lambda item: hashlib.sha256(item[0].encode()).hexdigest()): if owner != consumer: continue digest = hashlib.sha256(token.encode()).hexdigest() rows.append({ "consumer": owner, "api_key": token, "sha256_prefix": digest[:12], "storage": "CALLER_KEYS_JSON", }) return rows def _extract_token(request: Request) -> str | None: auth = request.headers.get("authorization") or "" if auth.lower().startswith("bearer "): return auth[7:].strip() return None def _caller_from_token(token: str | None) -> str | None: auth = _caller_auth(token) return auth.get("caller") if auth.get("ok") else None def _key_record_allows(digest: str, meta: dict[str, Any]) -> tuple[bool, str | None]: keys = meta.get("keys") or [] if not keys: return True, None matching = [k for k in keys if digest.startswith(str(k.get("sha256_prefix") or ""))] if not matching: return True, None now = int(time.time()) for key in matching: if key.get("status") == "revoked": return False, "caller_key_revoked" expires_at = int(key.get("expires_at") or 0) if expires_at and expires_at <= now: return False, "caller_key_expired" return True, None def _caller_auth(token: str | None) -> dict[str, Any]: if not token: return {"ok": False, "error_code": "caller_auth"} digest = hashlib.sha256(token.encode()).hexdigest() caller = CALLER_KEYS.get(token) storage = "CALLER_KEYS_JSON" if caller else None if not caller: caller = CALLER_KEY_HASHES.get(digest) storage = "CALLER_KEYS_SHA256_JSON" if caller else None if not caller: caller, store_ok = host_store.consumer_for_digest(digest) if not store_ok: return {"ok": False, "digest": digest, "error_code": "caller_auth_unavailable"} storage = "host_store" if caller else None if not caller: return {"ok": False, "error_code": "caller_auth"} # Authentication is an admission decision. Read only this caller's row; # dashboard aggregation still uses the full-table helper separately. issued, metadata_ok = host_store.get_consumer_key(caller) if not metadata_ok: return {"ok": False, "caller": caller, "digest": digest, "storage": storage, "error_code": "caller_auth_unavailable"} meta = _normalize_consumer_record(caller, issued) if meta.get("status") != "active": return {"ok": False, "caller": caller, "digest": digest, "storage": storage, "error_code": "caller_inactive"} allowed, key_error = _key_record_allows(digest, meta) if not allowed: return {"ok": False, "caller": caller, "digest": digest, "storage": storage, "error_code": key_error} return {"ok": True, "caller": caller, "digest": digest, "storage": storage, "meta": meta} async def _caller_auth_async(token: str | None) -> dict[str, Any]: """_caller_auth plus the external-control-plane fallback. Local key stores stay authoritative: a locally known key — including a revoked/inactive one — never falls through to the control plane; only a pure local miss consults it (and only when the feature is configured, see control_plane_client.enabled()). On a resolve hit the caller becomes the control-plane consumer name (tenant slug) and the plan's rate limits ride the returned meta; an explicit local record for that same name keeps precedence (operator kill-switch: mark the slug inactive locally).""" auth = await asyncio.to_thread(_caller_auth, token) if auth.get("ok") or auth.get("caller") or not token or not control_plane_client.enabled(): return auth digest = hashlib.sha256(token.encode()).hexdigest() resolved = await control_plane_client.resolve_key(digest) if resolved is None or not resolved.active or not resolved.consumer: return auth caller = f"{CONTROL_PLANE_CALLER_PREFIX}{resolved.consumer}" meta = await asyncio.to_thread(_consumer_meta, caller) if meta.get("keys") or caller in CALLER_KEYS.values() or caller in CALLER_KEY_HASHES.values(): # A local consumer (its own keys) already answers to this name: serving # the tenant under it would merge attribution, limits and budgets across # two principals. Fail closed until the operator renames one of them # (or sets CONTROL_PLANE_CALLER_PREFIX — which also renames the tenant's # usage caller). A key-less local record stays a kill-switch/override. control_plane_client.log_collision_once(caller) return {"ok": False, "caller": caller, "digest": digest, "storage": "control_plane", "error_code": "caller_name_collision"} if meta.get("status") != "active": return {"ok": False, "caller": caller, "digest": digest, "storage": "control_plane", "error_code": "caller_inactive"} if meta.get("rate_per_min") is None: meta["rate_per_min"] = resolved.rate_per_min if meta.get("burst") is None: meta["burst"] = resolved.burst out = {"ok": True, "caller": caller, "digest": digest, "storage": "control_plane", "meta": meta} if resolved.tenant_id is not None: out["tenant_id"] = resolved.tenant_id if resolved.scope_version == 2: out.update(scope_version=2, project_id=resolved.project_id, environment_id=resolved.environment_id) return out def _rate_ok(caller: str, meta: dict[str, Any] | None = None) \ -> tuple[bool, bool, float]: meta = meta or _consumer_meta(caller) rate_per_min = int(meta.get("rate_per_min") or RATE_PER_MIN) burst = int(meta.get("burst") or BURST) return host_store.consume_rate_token(caller, rate_per_min, burst) # Settles run off the event loop: a slow store must not stall every request on # this worker. Until a settle lands its reservation still counts, which only # errs toward rejecting; if it fails, the reservation expires (TTL). _SETTLE_POOL = concurrent.futures.ThreadPoolExecutor(max_workers=4, thread_name_prefix="budget-settle") _pending_settles: set[concurrent.futures.Future] = set() SETTLE_RETRY_DELAYS_S = (0.5, 2.0, 5.0, 15.0, 60.0) def _submit_settle(reservation_id: str, cost: float | None, caller: str) -> None: def settle() -> None: # Retried so a brief store outage does not lose the call's cost; a # reservation swept after its TTL still books its cost once. for delay in SETTLE_RETRY_DELAYS_S: _settled, store_ok = host_store.settle_subject_budget(reservation_id, cost) if store_ok: return _metric_store_error("key_budget") time.sleep(delay) _log({"event": "key_budget_settle_failed", "caller": caller}) future = _SETTLE_POOL.submit(settle) _pending_settles.add(future) future.add_done_callback(_pending_settles.discard) def drain_settles(timeout: float = 10.0) -> None: """Wait for in-flight settles (tests and graceful shutdown).""" concurrent.futures.wait(list(_pending_settles), timeout=timeout) def _requested_route_from(path: str, body: bytes | None) -> str | None: route = None if body: try: parsed = json.loads(body.decode("utf-8")) if isinstance(parsed, dict): route = parsed.get("model") or parsed.get("name") except Exception: route = None parts = [p for p in path.strip("/").split("/") if p] if len(parts) >= 3 and parts[1:] == ["v1", "chat", "completions"] and parts[0] != "v1": route = route or f"profile:{parts[0]}" # The router already treats a missing/empty OpenAI model as its current # default policy. Normalize admission to that same identity so a consumer # restricted to profile:default is not rejected before reaching the router. if not route and parts in (["v1", "chat", "completions"], ["v1", "responses"]): route = "profile:default" if not route and parts in (["v1", "decisions"], ["v1", "systemone"]): route = "policy:decisions" return str(route).strip() if route else None def _route_matches(pattern: str, route: str) -> bool: pattern = pattern.strip() if pattern in {"*", "all"} or pattern == route: return True if pattern.endswith("*") and route.startswith(pattern[:-1]): return True return False def _route_allowed(caller: str, route: str | None, meta: dict[str, Any] | None = None) -> bool: meta = meta if meta is not None else _consumer_meta(caller) allowed_routes = meta.get("allowed_routes") or [] if not allowed_routes: return True if not route: return False return any(_route_matches(pattern, route) for pattern in allowed_routes) def _log(event: dict) -> None: log.info(json.dumps(event, sort_keys=True, separators=(",", ":"))) def _b64e(raw: bytes) -> str: return base64.urlsafe_b64encode(raw).decode().rstrip("=") def _b64d(raw: str) -> bytes: return base64.urlsafe_b64decode(raw + "=" * (-len(raw) % 4)) def _make_dashboard_session(user: str, *, role: str = "admin", consumer: str | None = None, key_sha256: str | None = None) -> str: if not DASHBOARD_SESSION_SECRET: raise RuntimeError("DASHBOARD_SESSION_SECRET is not configured") payload = {"u": user, "r": role, "e": int(time.time()) + DASHBOARD_COOKIE_MAX_AGE} if consumer: payload["c"] = consumer if key_sha256: payload["k"] = key_sha256 body = _b64e(json.dumps(payload, separators=(",", ":"), sort_keys=True).encode()) sig = hmac.new(DASHBOARD_SESSION_SECRET.encode(), body.encode(), hashlib.sha256).hexdigest() return f"{body}.{sig}" def _dashboard_session_context(request: Request) -> dict[str, Any] | None: if not DASHBOARD_SESSION_SECRET: return None raw = request.cookies.get(DASHBOARD_COOKIE_NAME, "") if "." not in raw: return None body, sig = raw.rsplit(".", 1) expected = hmac.new(DASHBOARD_SESSION_SECRET.encode(), body.encode(), hashlib.sha256).hexdigest() if not control_plane_client.secret_equal(sig, expected): return None try: payload = json.loads(_b64d(body)) except Exception: return None if int(payload.get("e", 0)) < int(time.time()): return None user = str(payload.get("u", "")).strip() if not user: return None role = str(payload.get("r") or "admin").strip() if role == "consumer": consumer = str(payload.get("c", "")).strip() key_sha256 = str(payload.get("k", "")).strip().lower() if not consumer or not re.fullmatch(r"[a-f0-9]{64}", key_sha256): return None return {"role": "consumer", "user": user, "viewer": f"consumer:{consumer}", "consumer": consumer, "key_sha256": key_sha256} return {"role": "admin", "user": user, "viewer": f"dashboard:{user}", "consumer": None, "key_sha256": None} def _dashboard_session_user(request: Request) -> str | None: ctx = _dashboard_session_context(request) return str(ctx.get("viewer")) if ctx else None def _dashboard_password_ok(password: str) -> bool: if not DASHBOARD_PASSWORD_SHA256: return False got = hashlib.sha256(password.encode()).hexdigest() return control_plane_client.secret_equal(got, DASHBOARD_PASSWORD_SHA256) def _require_dashboard_context(request: Request) -> dict[str, Any] | None: if DASHBOARD_NO_AUTH: return {"role": "admin", "user": "local-dev", "viewer": "dashboard:local-dev", "consumer": None, "key_sha256": None} if DASHBOARD_TRUSTED_USER_HEADER: trusted_user = (request.headers.get(DASHBOARD_TRUSTED_USER_HEADER) or "").strip() trusted_secret = (request.headers.get("x-dashboard-trusted-secret") or "").strip() if trusted_user and DASHBOARD_TRUSTED_USER_SECRET and control_plane_client.secret_equal(trusted_secret, DASHBOARD_TRUSTED_USER_SECRET): return {"role": "admin", "user": trusted_user, "viewer": f"dashboard:{trusted_user}", "consumer": None, "key_sha256": None} return _dashboard_session_context(request) def _require_dashboard_auth(request: Request) -> str | None: ctx = _require_dashboard_context(request) return str(ctx.get("viewer")) if ctx else None def _require_admin_dashboard_auth(request: Request) -> tuple[dict[str, Any] | None, Response | None]: ctx = _require_dashboard_context(request) if not ctx: return None, JSONResponse(status_code=401, content={"error": {"message": "unauthorized dashboard caller", "type": "auth_error", "code": "dashboard_auth"}}) if ctx.get("role") != "admin": return None, JSONResponse(status_code=403, content={"error": {"message": "admin dashboard session required", "type": "auth_error", "code": "dashboard_admin_required"}}) return ctx, None def _require_admin_dashboard_caller(request: Request) -> tuple[str | None, Response | None]: # Privileged dashboard ops (reveal keys, manage codex, add/edit providers) # gate on the ADMIN role — which both the SSO trusted-header path and the # password login produce. There are no per-user tiers, so there is no extra # per-name gate: access control is whoever SSO/the password admits. Returns # the caller (viewer) string for audit, or an error Response. ctx, error = _require_admin_dashboard_auth(request) if error: return None, error return str(ctx.get("viewer")), None def _mask_remote(remote: str | None) -> str | None: if not remote: return None host = str(remote).strip() if not host: return None try: import ipaddress ip = ipaddress.ip_address(host) if ip.version == 4: parts = host.split(".") return ".".join(parts[:3] + ["0"]) + "/24" return str(ipaddress.ip_network(str(ip) + "/64", strict=False)) except Exception: return hashlib.sha256(host.encode()).hexdigest()[:12] def _user_agent_hash(request: Request) -> str | None: ua = (request.headers.get("user-agent") or "").strip() return hashlib.sha256(ua.encode()).hexdigest()[:12] if ua else None def _read_login_history() -> list[dict[str, Any]]: # Dashboard login audit, from the store now (#5: dashboard-logins.jsonl retired). return host_store.recent_logins(RECENT_LIMIT) def _record_dashboard_login(*, request: Request, role: str, user: str, consumer: str | None = None, key_sha256: str | None = None) -> None: key_prefix = key_sha256[:12] if key_sha256 and re.fullmatch(r"[a-f0-9]{64}", key_sha256) else None row = { "ts": int(time.time()), "event": "dashboard_login", "role": role, "viewer": f"consumer:{consumer}" if role == "consumer" and consumer else f"dashboard:{user}", "consumer": consumer, "key_sha256_prefix": key_prefix, "remote_masked": _mask_remote(request.client.host if request.client else None), "user_agent_hash": _user_agent_hash(request), } row = {k: v for k, v in row.items() if v is not None} host_store.insert_login(row) def _provider_raw_credential(name: str) -> tuple[str | None, str | None]: """(kind, raw value) of one provider's credential — env API keys come from the configured env var; codex OAuth comes from auth.json via CodexAuth (refreshed, so the copied token is currently valid). Only the admin-gated reveal endpoint may call this.""" try: cfg = _load_policy_config() except Exception: return None, None provider = (cfg.get("providers") or {}).get(name) if not isinstance(provider, dict): return None, None auth_env = str(provider.get("auth_env") or "").strip() if auth_env: val = os.getenv(auth_env, "").strip() return "env", (val or None) auth = provider.get("auth") if isinstance(provider.get("auth"), dict) else {} if auth.get("kind") == "oauth" or provider.get("api_kind") == "openai_codex": try: from codex_auth import CodexAuth token = CodexAuth(os.getenv("CODEX_AUTH_PATH") or None).access_token() return "oauth", (token or None) except Exception: return "oauth", None return None, None def _provider_credentials_snapshot(*, timeframe: str = "all", viewer_role: str = "admin") -> dict[str, Any]: """Admin-only provider credential/status/usage view. Never returns raw env values.""" if viewer_role != "admin": return {"rows": [], "privacy": {"admin_only": True, "hidden_for_consumer_sessions": True}} now = int(time.time()) since = None if timeframe == "runtime": since = int(_started_wall) elif timeframe == "1h": since = now - 3600 elif timeframe == "24h": since = now - 86400 elif timeframe == "7d": since = now - 7 * 86400 elif timeframe == "30d": since = now - 30 * 86400 try: cfg = _load_policy_config() except Exception: cfg = {"providers": {}} providers = cfg.get("providers") or {} prices = _price_table() counters: dict[str, dict[str, Any]] = defaultdict(_counter) last_event: dict[str, dict[str, Any]] = {} cost_by_provider: dict[str, float] = defaultdict(float) if timeframe == "recent": overlay = _snapshot_cached_compute( ("provider_recent",), lambda: host_store.usage_rows_page(limit=DASHBOARD_STATS_RECENT_LIMIT)) elif timeframe != "runtime": # SQL per-provider rollup over the window (was: load every retained row # and fold in Python), TTL-cached; only the not-yet-landed runtime rows # get folded on top — the old merge deduped landed rows anyway. cached = _snapshot_cache_get(("provider_stats", timeframe)) if cached is None: cached = host_store.usage_provider_stats(since_ts=since) _snapshot_cache_put(("provider_stats", timeframe), cached) for name, c in cached.items(): counters[name] = _counter_from_sql(c) if int(c.get("priced") or 0): cost_by_provider[name] = round(float(c.get("cost_usd") or 0.0), 6) last_event[name] = {"ts": c.get("last_ts"), "status": c.get("last_status"), "requested_model": c.get("last_route"), "model_family": c.get("last_model_family")} overlay = _unlanded_recent_requests(since) else: with _stats_lock: overlay = [dict(r) for r in _stats["recent"] if r.get("event") == "request"] if since is not None: overlay = [r for r in overlay if int(r.get("ts") or 0) >= since] with _stats_lock: runtime_by_provider = {k: _counter_snapshot(v) for k, v in sorted(_stats["by_provider"].items())} for row in overlay: provider = str(row.get("provider") or "unknown") cost, _meta = _cost_for_event(row, prices) _add_counter(counters[provider], row, cost) if cost is not None: cost_by_provider[provider] = round(cost_by_provider[provider] + cost, 6) ts = int(row.get("ts") or 0) if provider not in last_event or ts >= int(last_event[provider].get("ts") or 0): last_event[provider] = row rows = [] for name, provider in sorted(providers.items()): auth = provider.get("auth") if isinstance(provider.get("auth"), dict) else None auth_env = str(provider.get("auth_env") or "").strip() or None auth_kind = str(auth.get("kind")) if auth else ("env_api_key" if auth_env else "none") key_present = False key_fingerprint = None status = "missing" if auth_env: val = os.getenv(auth_env, "") key_present = bool(val.strip()) status = "configured" if key_present else "missing" key_fingerprint = hashlib.sha256(val.encode()).hexdigest()[:12] if key_present else None elif auth_kind == "none": status = "no_auth_required" elif auth_kind == "oauth": status = "oauth_configured" elif auth_kind: status = "configured" counter = _counter_snapshot(counters.get(name, _counter())) if timeframe in {"runtime", "recent"} and not counter.get("requests"): counter = runtime_by_provider.get(name, counter) latest = last_event.get(name, {}) rows.append({ "provider": name, "tier": provider.get("tier"), "api_kind": provider.get("api_kind"), "base_url": provider.get("base_url"), "auth_kind": auth_kind, "auth_env": auth_env, "credential_status": status, "key_present": key_present, "key_fingerprint": key_fingerprint, "usage": counter, "estimated_cost_usd": round(cost_by_provider.get(name, 0.0), 6), "last_status": latest.get("status"), "last_route": latest.get("requested_model") or latest.get("route"), "last_model_family": latest.get("model_family"), "last_seen": latest.get("ts") or counter.get("last_seen"), "notes": provider.get("notes"), }) known = {r["provider"] for r in rows} for name, counter in sorted(counters.items()): if name in known or name == "unknown": continue latest = last_event.get(name, {}) rows.append({ "provider": name, "tier": None, "api_kind": None, "base_url": None, "auth_kind": "unknown", "auth_env": None, "credential_status": "seen_in_usage_only", "key_present": None, "key_fingerprint": None, "usage": _counter_snapshot(counter), "estimated_cost_usd": round(cost_by_provider.get(name, 0.0), 6), "last_status": latest.get("status"), "last_route": latest.get("requested_model") or latest.get("route"), "last_model_family": latest.get("model_family"), "last_seen": latest.get("ts"), "notes": "Provider appeared in usage but is not in current config.", }) rows.sort(key=lambda r: (0 if r.get("credential_status") in {"configured", "oauth_configured", "no_auth_required"} else 1, -int((r.get("usage") or {}).get("requests") or 0), str(r.get("provider")))) return { "schema_version": 1, "kind": "router_provider_credentials", "generated_at": now, "timeframe": timeframe, "rows": rows, "privacy": { "admin_only": True, "raw_provider_keys_exposed": False, "full_key_hashes_exposed": False, "key_fingerprint": "sha256_prefix_12_when_env_key_present", }, } def _login_connections_snapshot(*, timeframe: str = "all", consumer: str | None = None, viewer_role: str = "admin") -> dict[str, Any]: if viewer_role != "admin": return {"rows": [], "recent_dashboard_logins": [], "privacy": {"admin_only": True, "hidden_for_consumer_sessions": True}} now = int(time.time()) since = None if timeframe == "runtime": since = int(_started_wall) elif timeframe == "1h": since = now - 3600 elif timeframe == "24h": since = now - 86400 elif timeframe == "7d": since = now - 7 * 86400 elif timeframe == "30d": since = now - 30 * 86400 dashboard_rows = list(_read_login_history()) with _stats_lock: runtime_recent = list(_stats["recent"]) dedup = {} for row in dashboard_rows: key = (row.get("ts"), row.get("viewer"), row.get("key_sha256_prefix"), row.get("remote_masked"), row.get("user_agent_hash")) dedup[key] = row dashboard_rows = list(dedup.values()) if since is not None: dashboard_rows = [r for r in dashboard_rows if int(r.get("ts") or 0) >= since] if consumer: dashboard_rows = [r for r in dashboard_rows if r.get("consumer") == consumer or r.get("viewer") == f"consumer:{consumer}"] grouped: dict[tuple[str, str], dict[str, Any]] = {} if timeframe == "recent": usage_rows = _snapshot_cached_compute( ("connections_recent", consumer), lambda: host_store.usage_rows_page( caller=consumer, limit=DASHBOARD_STATS_RECENT_LIMIT)) elif timeframe != "runtime": # SQL per-(caller, key-prefix) rollup over the window (was: load every # retained row), TTL-cached; the not-yet-landed runtime rows fold on top. cached = _snapshot_cache_get(("connections", timeframe, consumer)) if cached is None: cached = host_store.usage_connections(since_ts=since, caller=consumer) _snapshot_cache_put(("connections", timeframe, consumer), cached) for c in cached: prefix = str(c.get("prefix") or "unknown") name = str(c.get("caller") or "unknown") grouped[(name, prefix)] = { "kind": "api_key", "identity": name, "consumer": name, "key_sha256_prefix": prefix if prefix != "unknown" else None, "first_seen": c.get("first_seen"), "last_seen": c.get("last_seen"), "requests": int(c.get("requests") or 0), "errors": int(c.get("errors") or 0), "last_status": c.get("last_status"), "last_route": c.get("last_route"), "last_provider": c.get("last_provider")} usage_rows = _unlanded_recent_requests(since) else: usage_rows = [dict(r) for r in runtime_recent if r.get("event") == "request"] if since is not None: usage_rows = [r for r in usage_rows if int(r.get("ts") or 0) >= since] if consumer: usage_rows = [r for r in usage_rows if r.get("caller") == consumer] for row in usage_rows: caller = str(row.get("caller") or "unknown") prefix = str(row.get("key_sha256_prefix") or "").strip() or "unknown" key = (caller, prefix) item = grouped.setdefault(key, {"kind": "api_key", "identity": caller, "consumer": caller, "key_sha256_prefix": prefix if prefix != "unknown" else None, "first_seen": None, "last_seen": None, "requests": 0, "errors": 0, "last_status": None, "last_route": None, "last_provider": None}) ts = int(row.get("ts") or 0) item["requests"] += 1 if int(row.get("status") or 0) >= 400: item["errors"] += 1 item["first_seen"] = ts if not item["first_seen"] else min(int(item["first_seen"]), ts) if not item["last_seen"] or ts >= int(item["last_seen"]): item["last_seen"] = ts item["last_status"] = row.get("status") item["last_route"] = row.get("requested_model") or row.get("route") item["last_provider"] = row.get("provider") api_rows = list(grouped.values()) for item in api_rows: item["active_now"] = bool(item.get("last_seen") and now - int(item["last_seen"]) <= 3600) item["error_rate"] = round(item["errors"] / item["requests"], 4) if item["requests"] else 0.0 recent_logins = sorted(dashboard_rows, key=lambda r: int(r.get("ts") or 0), reverse=True)[:100] rows = sorted(api_rows, key=lambda r: int(r.get("last_seen") or 0), reverse=True) return { "rows": rows, "recent_dashboard_logins": recent_logins, "generated_at": now, "active_window_s": 3600, "privacy": { "admin_only": True, "raw_api_keys_exposed": False, "full_key_hashes_exposed": False, "remote_addresses_masked": True, "user_agents_hashed": True, }, } def _run_cost_backfill() -> int: """One-time (idempotent) stamping of cost_usd on unpriced `calls` rows so the SQL aggregates can trust SUM(cost_usd) — before this, every dashboard read re-priced those rows per row per load. Safe to run at every startup: rows the price table can't price stay NULL (they sum as 0, same as before).""" try: n = host_store.backfill_call_costs(_price_table()) if n: _log({"event": "cost_backfill", "rows_stamped": n}) return n except Exception as exc: # noqa: BLE001 — best-effort telemetry, never blocks startup logging.getLogger("auth_proxy").warning("cost backfill failed: %s", exc) return 0 _backfill_task: "asyncio.Task | None" = None @app.on_event("startup") async def startup() -> None: global _client, _probe_task, _backfill_task, _capacity # The ingress is the owner of the operational store (consumer keys, the ledger, # operator-config writes); all operator state lives in Postgres now (the legacy # JSON backfill was retired once prod confirmed the tables were populated). _capacity = (asyncio.Semaphore(MAX_INFLIGHT_REQUESTS) if MAX_INFLIGHT_REQUESTS else None) _client = httpx.AsyncClient( timeout=httpx.Timeout(90.0, connect=10.0), limits=httpx.Limits( max_connections=UPSTREAM_MAX_CONNECTIONS, max_keepalive_connections=UPSTREAM_MAX_KEEPALIVE_CONNECTIONS)) digests_ok = await asyncio.to_thread( host_store.upsert_consumer_key_digests, _exact_consumer_key_digests()) if not digests_ok: _metric_store_error("consumer_digest_startup") log.warning("consumer digest startup backfill failed; shared auth will fail closed") if SYNTHETIC_PROBES_ENABLED and ROUTE_HEALTH_ROUTES: _probe_task = asyncio.create_task(_synthetic_probe_loop()) # Off the event loop: batched UPDATEs, potentially many rows on first deploy. if RUN_COST_BACKFILL: _backfill_task = asyncio.create_task( asyncio.to_thread(_run_cost_backfill)) @app.on_event("shutdown") async def shutdown() -> None: global _probe_task await asyncio.to_thread(drain_settles) if _probe_task: _probe_task.cancel() with contextlib.suppress(asyncio.CancelledError): await _probe_task _probe_task = None if _client: await _client.aclose() await control_plane_client.close() @app.get("/healthz") async def healthz() -> Response: assert _client is not None try: r = await _client.get(f"{UPSTREAM}/healthz", timeout=5.0) return JSONResponse(status_code=r.status_code, content=r.json()) except Exception as exc: return JSONResponse(status_code=502, content={"ok": False, "error": str(exc)}) @app.get("/metrics") async def metrics() -> Response: return Response( content=_prometheus_metrics(), media_type="text/plain; version=0.0.4; charset=utf-8") @app.get("/favicon.ico") async def favicon() -> Response: return Response(status_code=204) @app.get("/dashboard") @app.get("/dashboard/") @app.get("/dashboard/provider-keys") @app.get("/dashboard/provider-keys/") async def dashboard() -> HTMLResponse: return HTMLResponse(_dashboard_html()) @app.post("/dashboard/api/login") @app.post("/dashboard/login") async def dashboard_login(request: Request) -> Response: try: data = await request.json() except Exception: data = {} password = str(data.get("password", "")) if isinstance(data, dict) else "" api_key = str(data.get("api_key", "")) if isinstance(data, dict) else "" if api_key.strip(): auth = _caller_auth(api_key.strip()) if not auth.get("ok"): return JSONResponse(status_code=401, content={"error": {"message": "invalid or inactive API key", "type": "auth_error", "code": auth.get("error_code") or "consumer_dashboard_login"}}) consumer = str(auth.get("caller") or "") digest = str(auth.get("digest") or "") _record_dashboard_login(request=request, role="consumer", user=consumer, consumer=consumer, key_sha256=digest) resp = JSONResponse(content={"ok": True, "role": "consumer", "user": consumer, "consumer": consumer, "max_age": DASHBOARD_COOKIE_MAX_AGE}) resp.set_cookie( DASHBOARD_COOKIE_NAME, _make_dashboard_session(consumer, role="consumer", consumer=consumer, key_sha256=digest), max_age=DASHBOARD_COOKIE_MAX_AGE, httponly=True, secure=True, samesite="lax", path=DASHBOARD_COOKIE_PATH, ) return resp if not _dashboard_password_ok(password): return JSONResponse(status_code=401, content={"error": {"message": "invalid dashboard password", "type": "auth_error", "code": "dashboard_login"}}) _record_dashboard_login(request=request, role="admin", user="admin") resp = JSONResponse(content={"ok": True, "role": "admin", "user": "admin", "max_age": DASHBOARD_COOKIE_MAX_AGE}) resp.set_cookie( DASHBOARD_COOKIE_NAME, _make_dashboard_session("admin"), max_age=DASHBOARD_COOKIE_MAX_AGE, httponly=True, secure=True, samesite="lax", path=DASHBOARD_COOKIE_PATH, ) return resp @app.post("/dashboard/api/logout") @app.post("/dashboard/logout") async def dashboard_logout() -> Response: resp = JSONResponse(content={"ok": True}) resp.delete_cookie(DASHBOARD_COOKIE_NAME, path=DASHBOARD_COOKIE_PATH) return resp def _load_policy_config() -> dict[str, Any]: """Load the Lua policy catalog for dashboard display only; never returns secrets.""" from lupa import LuaRuntime from llm_router_host import _to_py cfg_path = Path(DASHBOARD_POLICY_CONFIG_PATH) if not cfg_path.is_absolute(): cfg_path = Path.cwd() / cfg_path lua = LuaRuntime(unpack_returned_tuples=True) core_dir = (Path.cwd() / "core").resolve() lua.globals()["__policy_config_path"] = str(cfg_path.resolve()) lua.globals()["__policy_core_dir"] = str(core_dir) lua.execute("package.path = __policy_core_dir .. '/?.lua;' .. __policy_core_dir .. '/?/init.lua;' .. package.path") cfg = _to_py(lua.eval("dofile(__policy_config_path)")) or {} return _merge_provider_overlay(cfg) def _merge_provider_overlay(cfg: dict[str, Any]) -> dict[str, Any]: """Fold operator-added providers (the provider_overlays store) into the parsed catalog so the dashboard lists them like hand-configured ones.""" try: from provider_overlay import load_overlay overlay = load_overlay() except Exception: return cfg providers = cfg.setdefault("providers", {}) models = cfg.setdefault("models", {}) for pid, entry in (overlay.get("providers") or {}).items(): if pid in providers or not isinstance(entry, dict): continue providers[pid] = {k: v for k, v in entry.items() if k not in ("served_models", "added_at")} providers[pid].setdefault("api_kind", "openai_compatible") providers[pid].setdefault("discovery", "static") for sm in entry.get("served_models") or []: model = models.get((sm or {}).get("family")) if not isinstance(model, dict): continue served = model.setdefault("served_by", []) if not any(s.get("provider") == pid for s in served if isinstance(s, dict)): row = {"provider": pid} if sm.get("provider_model_id"): row["provider_model_id"] = sm["provider_model_id"] served.append(row) return cfg def _policy_file_profile_names() -> list[str]: policy_dir = Path(DASHBOARD_POLICY_DIR) if not policy_dir.is_absolute(): policy_dir = Path.cwd() / policy_dir names = {path.stem for path in policy_dir.glob("*.lua")} preferred = [name for name in ("edge", "medium", "dummy") if name in names] return preferred + sorted(names - set(preferred)) def _resolved_path(path_value: str) -> Path: path = Path(path_value) if not path.is_absolute(): path = Path.cwd() / path return path def _ranked_models_for_profile(profile_name: str) -> list[dict[str, Any]]: """Use the router itself to derive the profile's candidate population. The policy files are functional DSL sentences, not static `{quality_min=...}` tables. Deriving diagram rows from router.rank keeps the dashboard faithful to config.live.lua + metrics.live.lua + policies/*.lua. """ from llm_router_host import LLMRouterHost cfg_path = _resolved_path(DASHBOARD_POLICY_CONFIG_PATH) metrics_path = _resolved_path(DASHBOARD_POLICY_METRICS_PATH) host = LLMRouterHost( router_path=Path.cwd() / "core" / "router.lua", config_path=cfg_path, metrics_path=metrics_path if metrics_path.exists() else None, ) host.init() ranked, _rejected = host.rank({"profile": profile_name, "requirements": {"context": ROUTER_CONTEXT_LENGTH}}) grouped: dict[str, dict[str, Any]] = {} order: list[str] = [] for idx, row in enumerate(ranked, start=1): candidate = row.get("candidate") or {} family = candidate.get("model_family") or row.get("model_family") if not family: continue if family not in grouped: grouped[family] = { "name": family, "quality": candidate.get("quality_hint"), "capabilities": candidate.get("capabilities") or {}, "served_by": [], } order.append(family) grouped[family]["served_by"].append({ "order": len(grouped[family]["served_by"]) + 1, "rank": idx, "provider": candidate.get("provider_id"), "provider_model_id": candidate.get("served_model_id"), "provider_tier": candidate.get("tier"), "score": row.get("score"), }) return [grouped[name] for name in order] PROVIDER_HEALTH_WINDOW_S = int(os.getenv("PROVIDER_HEALTH_WINDOW_S", "900")) async def _fetch_router_runtime() -> dict[str, Any] | None: """Live breaker/disabled/EMA state from the router's /x/runtime. Returns None (→ health falls back to local signals) on any failure.""" if _client is None: return None try: r = await _client.get(f"{UPSTREAM}/x/runtime", timeout=3.0) if r.status_code == 200 and isinstance(r.json(), dict): return r.json() except Exception: return None return None async def _fetch_live_market() -> dict[str, Any] | None: """Full price book (per-family sellers + perf) from the router's /x/market. None on any failure — the Market tab shows the error.""" if _client is None: return None try: r = await _client.get(f"{UPSTREAM}/x/market", timeout=3.0) if r.status_code == 200 and isinstance(r.json(), dict): return r.json() except Exception: return None return None def _recent_provider_attempts(window_s: int = PROVIDER_HEALTH_WINDOW_S) -> dict[str, dict[str, Any]]: """Newest attempt outcome per provider from recent decision traces. _stats["recent"] is newest-first; within a row, decision_path is oldest-first, so scan it reversed and keep the first hit per provider.""" cutoff = time.time() - window_s out: dict[str, dict[str, Any]] = {} with _stats_lock: rows = list(_stats["recent"]) for row in rows: if not isinstance(row, dict) or row.get("event") != "request": continue ts = row.get("ts") or 0 if ts < cutoff: continue trace = row.get("decision_trace") if isinstance(row.get("decision_trace"), dict) else None for e in reversed((trace or {}).get("decision_path") or []): if not isinstance(e, dict) or e.get("event") != "attempted": continue pid = e.get("provider_id") if not pid or pid in out: continue out[pid] = { "ok": not e.get("error_kind"), "error_kind": e.get("error_kind"), "http_status": e.get("http_status"), "ts": ts, } pid = row.get("provider") if pid and pid not in out and 200 <= int(row.get("status") or 0) < 300: out[pid] = {"ok": True, "error_kind": None, "http_status": None, "ts": ts} return out def _runway_for(provider: dict[str, Any], balance: dict[str, Any] | None) -> str | None: # Runway thresholds are operator-tunable from the dashboard Config tab. if not balance or balance.get("value") is None: return None kind, value = balance.get("kind"), float(balance["value"]) if kind == "deposits_usdc": cap = provider.get("market_price_cap") or {} if not (float(cap.get("input", 0) or 0) or float(cap.get("output", 0) or 0)): return None # free buyer pays nobody; deposits are irrelevant return "empty" if value <= settings.get("antseed.runway_deposits_empty_usdc") \ else "low" if value < settings.get("antseed.runway_deposits_low_usdc") else "ok" if kind == "credits_usd": return "empty" if value <= settings.get("openrouter.runway_credits_empty_usd") \ else "low" if value < settings.get("openrouter.runway_credits_low_usd") else "ok" if kind == "quota_window": return "low" if value > settings.get("codex.runway_quota_low_fraction") else "ok" return None async def _attach_antseed_wallet(provider_keys: dict | None) -> None: """Enrich the AntSeed provider-credentials row with its buyer hot-wallet (address / deposits / runway), read LIVE from the market so the address always reflects the current identity. Lets the Provider keys tab show where to top up instead of a useless 'no auth required' — AntSeed needs no API key, it needs a funded wallet.""" if not provider_keys or not provider_keys.get("rows"): return market = await _fetch_live_market() w = (market or {}).get("wallet") if not w or not w.get("address"): return if w.get("deposits_available") is not None: try: reserved = float(w.get("deposits_reserved") or 0) except (TypeError, ValueError): reserved = 0.0 # `reserved` is the buyer's own deposit locked in active payment channels # (in use, returns to available as channels settle) — NOT spent and NOT # lost. Run the runway off the TOTAL so a fully-reserved wallet reads as # funded-in-use, not "empty · top up". total = float(w["deposits_available"]) + reserved w = {**w, "runway": _runway_for( {"market_price_cap": {"input": 1, "output": 1}}, {"kind": "deposits_usdc", "value": total})} for row in provider_keys["rows"]: if row.get("provider") == w.get("provider"): row["wallet"] = w break def _provider_health_for(provider: dict[str, Any], name: str, runtime: dict[str, Any] | None, recent: dict[str, Any]) -> dict[str, Any]: """Classify one provider: disconnected | failing | ok | idle. Carries the typed balance and a runway verdict; a paid provider with an empty balance is failing BEFORE any request hits the wall.""" rt = runtime or {} balance = (rt.get("balances") or {}).get(name) runway = _runway_for(provider, balance) def _result(state: str, reason: str | None) -> dict[str, Any]: return {"state": state, "reason": reason, "balance": balance, "runway": runway} auth = provider.get("auth") env_name = provider.get("auth_env") or (auth.get("env") if isinstance(auth, dict) else None) if env_name and not os.environ.get(env_name): return _result("disconnected", f"{env_name} not set") disabled = (rt.get("disabled_providers") or {}).get(name) if disabled: return _result("failing", f"disabled: {disabled}") last = recent.get(name) def _last_err() -> str | None: if not last or last.get("ok") or not last.get("error_kind"): return None kind = last["error_kind"] return f"{kind}({last['http_status']})" if last.get("http_status") else str(kind) breaker = (rt.get("circuit_breakers") or {}).get(name) or {} if breaker.get("open"): reason = "breaker open" err = _last_err() return _result("failing", f"{reason} · {err}" if err else reason) if runway == "empty": unit = "USDC" if (balance or {}).get("kind") == "deposits_usdc" else "USD" return _result("failing", f"balance empty ({balance['value']} {unit})") if last: if last.get("ok"): return _result("ok", None) return _result("failing", _last_err() or "recent error") return _result("idle", None) def _group_live_rank(rows: list[dict]) -> list[dict[str, Any]]: """Group /x/rank rows into the family-shaped structure the policy diagrams render (same shape as _ranked_models_for_profile, plus prices).""" grouped: dict[str, dict[str, Any]] = {} order: list[str] = [] for idx, row in enumerate(rows, start=1): family = row.get("model_family") if not family: continue if family not in grouped: grouped[family] = {"name": family, "quality": row.get("quality"), "capabilities": {}, "served_by": []} order.append(family) grouped[family]["served_by"].append({ "order": len(grouped[family]["served_by"]) + 1, "rank": idx, "provider": row.get("provider"), "provider_model_id": row.get("served_model_id"), "provider_tier": row.get("tier"), "score": row.get("score"), "price_in": row.get("price_in"), "price_out": row.get("price_out"), "discovery": row.get("discovery"), }) return [grouped[name] for name in order] async def _fetch_live_ranks(profile_names: list[str]) -> dict[str, list[dict]]: """Rank rows per profile from the live router; missing/failed -> absent (the snapshot falls back to its local reconstruction, labeled).""" out: dict[str, list[dict]] = {} if _client is None: return out for name in profile_names: try: r = await _client.get(f"{UPSTREAM}/x/rank", params={"profile": name}, timeout=3.0) if r.status_code == 200 and isinstance(r.json(), dict): out[name] = r.json().get("ranked") or [] except Exception: continue return out def _policy_catalog_snapshot(runtime: dict[str, Any] | None = None, live_ranks: dict[str, list[dict]] | None = None) -> dict[str, Any]: cfg = _load_policy_config() providers = cfg.get("providers") or {} models = cfg.get("models") or {} profiles = cfg.get("profiles") or {} retry_policies = cfg.get("retry_policies") or {} recent_attempts = _recent_provider_attempts() provider_rows = [] for name, provider in sorted(providers.items()): auth = provider.get("auth") auth_kind = auth.get("kind") if isinstance(auth, dict) else ("configured" if provider.get("auth_env") else "none") provider_rows.append({ "name": name, "tier": provider.get("tier"), "api_kind": provider.get("api_kind"), "base_url": provider.get("base_url"), "auth": auth_kind, "notes": provider.get("notes"), "health": _provider_health_for(provider, name, runtime, recent_attempts), }) model_rows = [] for name, model in sorted(models.items()): served_by = [] for idx, candidate in enumerate(model.get("served_by") or [], start=1): provider_id = candidate.get("provider") served_by.append({ "order": idx, "provider": provider_id, "provider_model_id": candidate.get("provider_model_id"), "provider_tier": (providers.get(provider_id) or {}).get("tier"), }) model_rows.append({ "name": name, "quality": model.get("static_quality_hint"), "capabilities": model.get("capabilities") or {}, "served_by": served_by, }) # All config profiles (declarative or file-based). With tiers removed this is # just the `default` fallback; per-call policies arrive as policy_ir. file_names = _policy_file_profile_names() policy_profile_names = [n for n in file_names if n in profiles] + \ [n for n in profiles if n not in file_names] profile_rows = [] for name in policy_profile_names: profile = profiles.get(name) or {} live = (live_ranks or {}).get(name) if live: ranked_models = _group_live_rank(live) rank_source = "router" note = "Live ranking from the router (/x/rank): live prices, breakers, marketplace offers." else: ranked_models = _ranked_models_for_profile(name) rank_source = "reconstructed" note = "Derived from router.rank using config.live.lua and metrics.live.lua." profile_rows.append({ "name": name, "filter": "policies/%s.lua" % name, "weights": {}, "retry_policy": profile.get("retry_policy"), "models": ranked_models, "candidate_count": sum(len(model.get("served_by") or []) for model in ranked_models), "selection_note": note, "rank_source": rank_source, }) cfg_path = _resolved_path(DASHBOARD_POLICY_CONFIG_PATH) metrics_path = _resolved_path(DASHBOARD_POLICY_METRICS_PATH) return { "providers": provider_rows, "models": model_rows, "profiles": profile_rows, "profile_source": "policy_files", "policy_files": ["policies/%s.lua" % name for name in policy_profile_names if name in file_names], "retry_policies": retry_policies, "source": str(cfg_path.resolve()), "metrics_source": str(metrics_path.resolve()) if metrics_path.exists() else None, "generated_at": int(time.time()), } @app.get("/dashboard/api/stats") async def dashboard_stats(request: Request) -> Response: ctx = _require_dashboard_context(request) if not ctx: return JSONResponse(status_code=401, content={"error": {"message": "unauthorized dashboard caller", "type": "auth_error", "code": "dashboard_auth"}}) caller = str(ctx.get("viewer")) assert _client is not None upstream_health: dict[str, Any] upstream_status = 0 try: r = await _client.get(f"{UPSTREAM}/healthz", timeout=5.0) upstream_status = r.status_code upstream_health = r.json() except Exception as exc: upstream_health = {"ok": False, "error": str(exc)} requested_consumer = (request.query_params.get("consumer") or "").strip() or None consumer = str(ctx.get("consumer") or "").strip() or requested_consumer key_sha256 = str(ctx.get("key_sha256") or "").strip() or None role = str(ctx.get("role") or "admin") if (unavailable := await _consumer_view_unavailable(role)) is not None: return unavailable # _stats_snapshot performs synchronous PostgreSQL aggregation. Keep it off # Uvicorn's event loop so a slow query cannot stall health checks and all # proxied traffic handled by this worker. snap = await asyncio.to_thread( _stats_snapshot, viewer=caller, upstream_status=upstream_status, upstream_health=upstream_health, consumer=consumer, timeframe=_dashboard_timeframe(request.query_params.get("timeframe") or request.query_params.get("window")), key_sha256=key_sha256, viewer_role=role, provider=request.query_params.get("provider"), model=request.query_params.get("model")) await _attach_antseed_wallet(snap.get("provider_keys")) payload = await asyncio.to_thread(json.dumps, snap, separators=(",", ":")) return Response(content=payload, media_type="application/json") async def _dashboard_full_snapshot(request: Request) -> Response: """Full sanitized dashboard state for quick external evaluation. Combines the live in-memory usage/route snapshot with the policy catalog. This intentionally does not expose raw API keys, env values, bearer tokens, or provider auth material. """ ctx = _require_dashboard_context(request) if not ctx: return JSONResponse(status_code=401, content={"error": {"message": "unauthorized dashboard caller", "type": "auth_error", "code": "dashboard_auth"}}) caller = str(ctx.get("viewer")) assert _client is not None upstream_health: dict[str, Any] upstream_status = 0 try: r = await _client.get(f"{UPSTREAM}/healthz", timeout=5.0) upstream_status = r.status_code upstream_health = r.json() except Exception as exc: upstream_health = {"ok": False, "error": str(exc)} requested_consumer = (request.query_params.get("consumer") or "").strip() or None consumer = str(ctx.get("consumer") or "").strip() or requested_consumer key_sha256 = str(ctx.get("key_sha256") or "").strip() or None role = str(ctx.get("role") or "admin") if (unavailable := await _consumer_view_unavailable(role)) is not None: return unavailable stats = await asyncio.to_thread( _stats_snapshot, viewer=caller, upstream_status=upstream_status, upstream_health=upstream_health, consumer=consumer, timeframe=_dashboard_timeframe(request.query_params.get("timeframe") or request.query_params.get("window")), key_sha256=key_sha256, viewer_role=role) try: policies = _policy_catalog_snapshot() if role == "admin" else {"providers": [], "models": [], "profiles": [], "retry_policies": {}, "consumer_visible": False, "generated_at": int(time.time())} policy_error = None except Exception as exc: policies = {"providers": [], "models": [], "profiles": [], "retry_policies": {}, "generated_at": int(time.time())} policy_error = {"message": str(exc), "type": "policy_catalog_error", "code": "policy_catalog"} content = { "schema_version": 1, "kind": "router_dashboard_full_snapshot", "generated_at": int(time.time()), "viewer": caller, "viewer_role": role, "selected_consumer": stats.get("selected_consumer"), "selected_key_sha256_prefix": stats.get("selected_key_sha256_prefix"), "sections": { "overview": { "uptime_s": stats.get("uptime_s"), "rate_limit": stats.get("rate_limit"), "upstream": stats.get("upstream"), "totals": stats.get("totals"), "health_summary": stats.get("health_summary"), }, "consumers": { "consumers": stats.get("consumers"), "keys": stats.get("keys"), "by_caller": stats.get("by_caller"), }, "routing": { "route_health": stats.get("route_health"), "health_summary": stats.get("health_summary"), "by_route": stats.get("by_route"), "by_provider": stats.get("by_provider"), "by_model_family": stats.get("by_model_family"), "by_served_model": stats.get("by_served_model"), "by_status": stats.get("by_status"), }, "policies": policies, "activity": { "recent": stats.get("recent"), }, "logins": stats.get("logins"), "provider_keys": stats.get("provider_keys"), }, "raw": { "stats": stats, "policies": policies, }, "errors": [policy_error] if policy_error else [], "security": { "sanitized": True, "raw_api_keys_exposed": False, "provider_credentials_exposed": False, }, } return JSONResponse(content=content) @app.get("/dashboard/api/full") async def dashboard_full(request: Request) -> Response: return await _dashboard_full_snapshot(request) @app.get("/dashboard/api/evaluate") async def dashboard_evaluate(request: Request) -> Response: return await _dashboard_full_snapshot(request) @app.get("/dashboard/api/policies") async def dashboard_policies(request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error caller = str(ctx.get("viewer")) try: runtime = await _fetch_router_runtime() live_ranks = await _fetch_live_ranks(_policy_file_profile_names()) return JSONResponse(content=_policy_catalog_snapshot(runtime=runtime, live_ranks=live_ranks)) except Exception as exc: return JSONResponse(status_code=500, content={"error": {"message": str(exc), "type": "policy_catalog_error", "code": "policy_catalog"}}) @app.get("/dashboard/api/market") async def dashboard_market(request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error data = await _fetch_live_market() if data is None: return JSONResponse(status_code=502, content={"error": { "message": "router /x/market unavailable", "type": "market_error", "code": "market"}}) return JSONResponse(content=data) SKILL_PATH = Path(__file__).resolve().parent / "SKILL.md" OPTIMIZATION_SKILL_PATH = SKILL_PATH.parent / "skills" / "policy-optimization" / "SKILL.md" SKILL_MARKER = "" FIELDS_MARKER = "" def _catalog_table_markdown(market: dict | None) -> str: """The family-level catalog as a markdown table — the unit a Σ_pol policy targets (gate/score on families and fields, not individual sellers). Benchmarks render as 0–100 with catalog rank; prices are the cheapest seller per family.""" families = (market or {}).get("families") or [] if not families: return "_(live catalog unavailable — target the field vocabulary above " \ "and confirm families with `POST /x/rank`)_" def bench(m: dict, key: str) -> str: v = m.get(key) if not isinstance(v, (int, float)): return "—" cell = f"{round(v * 100)}" rank = m.get(key + "_rank") if isinstance(rank, (int, float)) and rank < 1e8: cell += f" (#{int(rank)})" return cell def price(rows: list, field: str) -> str: vals = [r.get(field) for r in rows if isinstance(r.get(field), (int, float))] return f"{min(vals):g}" if vals else "—" cap_labels = [("cap_tools", "tools"), ("cap_reasoning", "reason"), ("cap_structured_outputs", "struct"), ("cap_seed", "seed"), ("cap_logprobs", "logprobs")] # (sigma-pol/v2) no "Quality" column: quality/quality_hint were removed as # observable fields (they meant nothing — a hand-assigned static hint), so a # policy can't gate/score on them. The real model-goodness signals are the # benchmarks below. out = ["| Family | Intel | Coding | Agentic | Arena | Modalities | Caps | $in/Mtok | $out/Mtok | Sellers |", "|---|---|---|---|---|---|---|---|---|---|"] for f in families: m = f.get("meta") or {} rows = f.get("rows") or [] mods = "/".join(k for k in ("image", "audio", "file", "video") if m.get("in_" + k)) or "—" caps = "/".join(label for key, label in cap_labels if m.get(key)) or "—" out.append( f"| `{f.get('family')}` " f"| {bench(m, 'bench_intelligence')} | {bench(m, 'bench_coding')} " f"| {bench(m, 'bench_agentic')} | {bench(m, 'bench_arena')} " f"| {mods} | {caps} | {price(rows, 'price_in')} " f"| {price(rows, 'price_out')} | {f.get('sellers_total', '—')} |") return "\n".join(out) def _field_vocabulary_markdown(fields: list | None) -> str: """The observable field vocabulary as a markdown table, derived live from the core's own schema (`/x/fields` → `host.field_schema()`). This is the one authoritative list of fields a policy may gate/score over: keeping it out of the static prose means a field appended to the core (the anti-telos says expressiveness grows through the field vocabulary, not ad-hoc ops) shows up here automatically instead of silently diverging.""" if not fields: return "_(field schema unavailable — the core defines the vocabulary; " \ "fetch it from `GET /x/fields` on this host)_" core = sorted((f for f in fields if f.get("core")), key=lambda f: f.get("name", "")) ext = sorted((f for f in fields if not f.get("core")), key=lambda f: f.get("name", "")) out = ["| Field | Sort | Scope | Group |", "|---|---|---|---|"] for f in core + ext: scope = "core" if f.get("core") else "host" out.append(f"| `{f.get('name')}` | {f.get('sort', '—')} | {scope} " f"| {f.get('group', '—')} |") return "\n".join(out) def _render_skill(market: dict | None, fields: list | None = None, *, path: Path | None = None) -> str: """The committed SKILL.md with the live catalog table AND the live field vocabulary injected at their markers — a self-contained doc to load into any assistant. The catalog and the vocabulary are both derived from the host/core at download time so the guide cannot drift from what the host actually serves.""" text = (path or SKILL_PATH).read_text() vocab = ("The fields below are read live from this host's core schema " "(`GET /x/fields`); `core` fields exist on every conforming host, " "`host` fields are this host's registered extensions. A field's " "presence does not prove every route has measured data.\n\n" + _field_vocabulary_markdown(fields)) if FIELDS_MARKER in text: text = text.replace(FIELDS_MARKER, vocab, 1) catalog = ("## Live catalog (this host)\n\n" "Catalog snapshot at download time, not a guarantee of eligible " "or reachable routes. Preview with the intended protocol and " "requirements before calling. Benchmarks are 0–100 with catalog rank in parentheses " "(1 = best); prices are the cheapest seller per family in " "USD/Mtok.\n\n" + _catalog_table_markdown(market)) if SKILL_MARKER in text: return text.replace(SKILL_MARKER, catalog, 1) return text.rstrip() + "\n\n" + catalog + "\n" async def _fetch_live_fields() -> list | None: """The observable field vocabulary from the router's /x/fields (core + this host's extensions). None on any failure — the guide then points the reader at /x/fields instead of inventing a list.""" if _client is None: return None try: r = await _client.get(f"{UPSTREAM}/x/fields", timeout=3.0) if r.status_code == 200 and isinstance(r.json(), dict): return r.json().get("fields") except Exception: return None return None @app.get("/dashboard/api/skill") async def dashboard_skill(request: Request, name: str = "policy-authoring") -> Response: """Download the selected agent guide with the live catalog and vocabulary. Defaults to policy authoring for existing dashboard clients.""" ctx, error = _require_admin_dashboard_auth(request) if error: return error return await _skill_response(name) @app.get("/skill") async def consumer_skill(request: Request, name: str = "policy-authoring") -> Response: """Download the selected SKILL.md with live catalog and field vocabulary, but authenticated by a CONSUMER KEY (Bearer) instead of a dashboard session — so an agent can fetch the guide for the host it routes through, using the same credential it calls /v1/* with, no dashboard login. Reuses the same renderer and the same key auth (revoked/expired/inactive keys are rejected).""" auth = request.headers.get("authorization") or "" token = auth[7:].strip() if auth.lower().startswith("bearer ") else auth.strip() if not (await _caller_auth_async(token)).get("ok"): return JSONResponse(status_code=401, content={"error": { "message": "invalid or inactive API key", "type": "auth_error", "code": "consumer_auth"}}) return await _skill_response(name) async def _skill_response(name: str) -> Response: # Select only committed files; never turn a query parameter into a path. paths = {"policy-authoring": SKILL_PATH, "policy-optimization": OPTIMIZATION_SKILL_PATH} if name not in paths: return JSONResponse(status_code=404, content={"error": { "message": "Unknown skill. Choose policy-authoring or policy-optimization.", "code": "skill_not_found"}}) try: text = _render_skill(await _fetch_live_market(), await _fetch_live_fields(), path=paths[name]) except OSError: return JSONResponse(status_code=503, content={"error": { "message": "Skill file unavailable on this host.", "code": "skill_unavailable"}}) return Response(content=text, media_type="text/markdown; charset=utf-8", headers={"Content-Disposition": "attachment; filename=SKILL.md"}) # ---- consumer-key validation surface --------------------------------------- # SKILL.md tells an agent to dry-run / identify a term WITHOUT spending via the # router's /x/* admission endpoints (normalize, rank, fields, build). Those live # on the router; the ingress only reached them server-side (dashboard), so a # consumer-key holder — the exact audience of /skill — got 404 and the documented # "admit before you run" workflow was unusable. Expose the READ-ONLY / no-spend # ones here under the same key auth as /skill. None execute a call, move money, # or mutate state: they admit/normalize a term, preview the ranking, or list the # field vocabulary — all derived from the live catalog the skill already shows. async def _consumer_x_proxy(request: Request, method: str, path: str) -> Response: if not (await _caller_auth_async(_extract_token(request))).get("ok"): return JSONResponse(status_code=401, content={"error": { "message": "invalid or inactive API key", "type": "auth_error", "code": "consumer_auth"}}) if _client is None: return JSONResponse(status_code=502, content={"error": { "message": "router client unavailable", "type": "router_error", "code": "router"}}) params = dict(request.query_params) try: if method == "POST": try: body = await request.json() except Exception: # noqa: BLE001 — a malformed body becomes an empty POST body = {} r = await _client.post(f"{UPSTREAM}{path}", json=body, params=params, timeout=10.0) else: r = await _client.get(f"{UPSTREAM}{path}", params=params, timeout=10.0) except Exception as exc: # noqa: BLE001 return JSONResponse(status_code=502, content={"error": { "message": f"router {path} unreachable: {exc}", "type": "router_error", "code": "router"}}) try: return JSONResponse(status_code=r.status_code, content=r.json()) except Exception: # noqa: BLE001 — pass through a non-JSON router body verbatim return Response(status_code=r.status_code, content=r.text, media_type=r.headers.get("content-type", "application/json")) @app.get("/x/fields") async def consumer_x_fields(request: Request) -> Response: return await _consumer_x_proxy(request, "GET", "/x/fields") @app.get("/x/policy/templates") async def consumer_x_policy_templates(request: Request) -> Response: return await _consumer_x_proxy(request, "GET", "/x/policy/templates") @app.post("/x/policy/templates/{template_id}") async def consumer_x_policy_template(request: Request, template_id: str) -> Response: if template_id not in {item["id"] for item in template_catalog()}: # Keep the forwarded path bounded, including encoded path fragments. if not (await _caller_auth_async(_extract_token(request))).get("ok"): return JSONResponse(status_code=401, content={"error": { "message": "invalid or inactive API key", "code": "consumer_auth"}}) return JSONResponse(status_code=404, content={"error": { "message": "Unknown policy template", "code": "template_not_found"}}) return await _consumer_x_proxy(request, "POST", f"/x/policy/templates/{template_id}") @app.post("/x/policy/normalize") async def consumer_x_policy_normalize(request: Request) -> Response: return await _consumer_x_proxy(request, "POST", "/x/policy/normalize") @app.post("/x/flow/normalize") async def consumer_x_flow_normalize(request: Request) -> Response: return await _consumer_x_proxy(request, "POST", "/x/flow/normalize") @app.post("/x/policy/build") async def consumer_x_policy_build(request: Request) -> Response: return await _consumer_x_proxy(request, "POST", "/x/policy/build") @app.get("/x/rank") async def consumer_x_rank_get(request: Request) -> Response: return await _consumer_x_proxy(request, "GET", "/x/rank") @app.post("/x/rank") async def consumer_x_rank_post(request: Request) -> Response: return await _consumer_x_proxy(request, "POST", "/x/rank") async def _router_post_json(path: str, payload: dict) -> Response: """Admin-dashboard passthrough to the shim's policy endpoints. The shim (and behind it the core) is the validator; this just forwards bytes and status — including the 400s the builder UI needs to render verbatim.""" if _client is None: return JSONResponse(status_code=503, content={"error": { "message": "router client not ready", "type": "router_error", "code": "upstream"}}) try: r = await _client.post(f"{UPSTREAM}{path}", json=payload, timeout=10.0) return Response(content=r.content, status_code=r.status_code, media_type="application/json") except Exception: return JSONResponse(status_code=502, content={"error": { "message": f"router {path} unavailable", "type": "router_error", "code": "upstream"}}) async def _router_lines_call(method: str, path: str, payload: dict | None = None) -> Response: """Admin-dashboard passthrough to the router's model-line endpoints, which sit behind the internal secret. Forwards status and body verbatim so a lost compare-and-set (409) reaches the operator as such.""" if _client is None: return JSONResponse(status_code=503, content={"error": { "message": "router client not ready", "type": "router_error", "code": "upstream"}}) try: r = await _client.request(method, f"{UPSTREAM}{path}", json=payload, headers=_router_admin_headers(), timeout=10.0) return Response(content=r.content, status_code=r.status_code, media_type="application/json") except Exception: return JSONResponse(status_code=502, content={"error": { "message": f"router {path} unavailable", "type": "router_error", "code": "upstream"}}) @app.get("/dashboard/api/lines") async def dashboard_model_lines(request: Request) -> Response: """Every model line with its current head, seed and recent audit events.""" ctx, error = _require_admin_dashboard_auth(request) if error: return error return await _router_lines_call("GET", "/x/lines") async def _dashboard_model_line_move(line: str, action: str, request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error if not re.fullmatch(r"[a-z][a-z0-9_]{0,47}", line): return JSONResponse(status_code=400, content={"error": { "message": "invalid line name", "type": "invalid_request_error", "code": "model_line"}}) try: body = await request.json() except Exception: body = None if not isinstance(body, dict): return JSONResponse(status_code=400, content={"error": { "message": "JSON body must be an object", "type": "invalid_request_error", "code": "invalid_json"}}) # The audit actor is the authenticated dashboard viewer, never a body field. body = {**body, "actor": str(ctx.get("viewer"))} return await _router_lines_call("POST", f"/x/lines/{line}/{action}", body) @app.post("/dashboard/api/lines/{line}/promote") async def dashboard_model_line_promote(line: str, request: Request) -> Response: return await _dashboard_model_line_move(line, "promote", request) @app.post("/dashboard/api/lines/{line}/rollback") async def dashboard_model_line_rollback(line: str, request: Request) -> Response: return await _dashboard_model_line_move(line, "rollback", request) def _backtest_timeframe_options(value: Any) -> dict[str, Any]: timeframe = _dashboard_timeframe(value or "7d") if timeframe == "runtime": return {"timeframe": timeframe, "since_ts": int(_started_wall)} if timeframe == "all": return {"timeframe": timeframe, "since_ts": None} opts = _dashboard_timeframe_options(timeframe) return {"timeframe": timeframe, "since_ts": opts.get("since")} def _json_number(value: Any) -> float | None: if isinstance(value, (int, float)): return float(value) try: text = str(value).strip() if text: return float(text) except (TypeError, ValueError): pass return None def _rank_candidate(row: dict[str, Any]) -> dict[str, Any]: cand = row.get("candidate") return cand if isinstance(cand, dict) else {} def _rank_provider(row: dict[str, Any]) -> str | None: cand = _rank_candidate(row) value = row.get("provider") or row.get("provider_id") or cand.get("provider_id") or cand.get("provider") return str(value) if value else None def _rank_family(row: dict[str, Any]) -> str | None: cand = _rank_candidate(row) value = row.get("model_family") or row.get("model") or cand.get("model_family") or cand.get("model") return str(value) if value else None def _rank_model(row: dict[str, Any]) -> str | None: cand = _rank_candidate(row) value = row.get("served_model_id") or row.get("provider_model_id") \ or cand.get("served_model_id") or cand.get("provider_model_id") \ or _rank_family(row) return str(value) if value else None def _rank_prices(row: dict[str, Any], family: str, provider: str | None, prices: dict[tuple[str, str], dict[str, float]]) -> tuple[float | None, float | None]: cand = _rank_candidate(row) offer = cand.get("offer") if isinstance(cand.get("offer"), dict) else {} pin = _json_number(row.get("price_in")) if pin is None: pin = _json_number(cand.get("price_in")) if pin is None: pin = _json_number(row.get("price_in_usd_per_mtok")) if pin is None: pin = _json_number(cand.get("price_in_usd_per_mtok")) if pin is None: pin = _json_number(offer.get("price_in_usd_per_mtok")) pout = _json_number(row.get("price_out")) if pout is None: pout = _json_number(cand.get("price_out")) if pout is None: pout = _json_number(row.get("price_out_usd_per_mtok")) if pout is None: pout = _json_number(cand.get("price_out_usd_per_mtok")) if pout is None: pout = _json_number(offer.get("price_out_usd_per_mtok")) fallback = prices.get((family, provider or "")) if fallback: if pin is None: pin = fallback.get("input") if pout is None: pout = fallback.get("output") return pin, pout def _counterfactual_cost(tokens_in: int, tokens_out: int, price_in: float | None, price_out: float | None) -> float | None: if tokens_in and price_in is None: return None if tokens_out and price_out is None: return None cost = (tokens_in / 1_000_000.0) * (price_in or 0.0) \ + (tokens_out / 1_000_000.0) * (price_out or 0.0) return round(max(0.0, cost), 6) def _backtest_candidate_summary(row: dict[str, Any], family: str, prices: dict[tuple[str, str], dict[str, float]]) -> dict[str, Any]: provider = _rank_provider(row) pin, pout = _rank_prices(row, family, provider, prices) return { "provider": provider, "model": _rank_model(row), "price_in": pin, "price_out": pout, "score": row.get("score"), } def _backtest_rank_error_kind(data: dict[str, Any]) -> str: error = data.get("error") if isinstance(data.get("error"), dict) else {} kind = error.get("kind") or data.get("kind") \ or error.get("type") or data.get("type") \ or error.get("code") or data.get("code") or "rank_error" return str(kind) async def _rank_policy_for_backtest(policy_ir: list, family: str) -> tuple[int, dict[str, Any]]: if _client is None: return 503, {"error": {"message": "router client not ready", "type": "router_error", "code": "upstream"}} payload = {"policy_ir": policy_ir, "requirements": {"context": ROUTER_CONTEXT_LENGTH, "model_family": family}} try: r = await _client.post(f"{UPSTREAM}/x/rank", json=payload, timeout=10.0) try: data = r.json() except Exception: # noqa: BLE001 data = {"error": {"message": r.text, "type": "router_error", "code": "rank"}} return r.status_code, data if isinstance(data, dict) else {} except Exception as exc: # noqa: BLE001 return 502, {"error": {"message": f"router /x/rank unavailable: {exc}", "type": "router_error", "code": "upstream"}} @app.post("/dashboard/api/policy/backtest") async def dashboard_policy_backtest(request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error try: body = await request.json() except Exception: body = {} if not isinstance(body, dict): body = {} if "flow_ir" in body: return JSONResponse(status_code=400, content={"error": { "message": "Σ_flow / flow_ir backtesting is out of scope; provide a Σ_pol policy_ir term.", "type": "invalid_request_error", "code": "flow_backtest_unsupported"}}) policy_ir = body.get("policy_ir") if isinstance(policy_ir, list) and policy_ir and policy_ir[0] == "flow": return JSONResponse(status_code=400, content={"error": { "message": "Σ_flow terms are out of scope for policy backtest; provide a Σ_pol policy_ir term.", "type": "invalid_request_error", "code": "flow_backtest_unsupported"}}) if not isinstance(policy_ir, list): return JSONResponse(status_code=400, content={"error": { "message": "policy_ir must be a Σ_pol term array", "type": "invalid_request_error", "code": "policy_ir_required"}}) window_opts = _backtest_timeframe_options(body.get("timeframe") or "7d") consumer = str(body.get("consumer") or "").strip() or None grouped = host_store.policy_backtest_groups( since_ts=window_opts["since_ts"], caller=consumer, limit=50) prices = _price_table() rank_cache: dict[str, dict[str, Any]] = {} groups = [] actual_total = 0.0 backtest_total = 0.0 unroutable_requests = 0 rank_error_requests = 0 rank_error_groups = 0 rank_successes = 0 first_rank_error: tuple[int, dict[str, Any]] | None = None unpriced_groups = 0 grouped_groups = grouped.get("groups") or [] rank_routes = list(dict.fromkeys( str(group.get("route") or "unknown") for group in grouped_groups)) rank_results: dict[str, tuple[int, dict[str, Any]]] = {} if rank_routes: rank_semaphore = asyncio.Semaphore(8) async def _rank_route(route: str) -> tuple[str, int, dict[str, Any]]: async with rank_semaphore: status, data = await _rank_policy_for_backtest(policy_ir, route) return route, status, data for route, status, data in await asyncio.gather(*( _rank_route(route) for route in rank_routes)): rank_results[route] = (status, data) if status < 400: rank_cache[route] = data rank_successes += 1 for group in grouped_groups: route = str(group.get("route") or "unknown") requests = int(group.get("requests") or 0) tokens_in = int(group.get("tokens_in") or 0) tokens_out = int(group.get("tokens_out") or 0) actual_cost = round(float(group.get("actual_cost_usd") or 0.0), 6) providers = group.get("providers") if isinstance(group.get("providers"), dict) else {} provider_latency = group.get("provider_latency_ms") \ if isinstance(group.get("provider_latency_ms"), dict) else {} status, data = rank_results[route] if status >= 400: if first_rank_error is None: first_rank_error = (status, data) rank_error_groups += 1 rank_error_requests += requests groups.append({ "route": route, "requests": requests, "tokens_in": tokens_in, "tokens_out": tokens_out, "actual": {"cost_usd": actual_cost, "providers": providers}, "backtest": {"winner_provider": None, "winner_model": None, "cost_usd": None, "admitted": 0, "fallbacks": []}, "latency": {"actual_avg_ms": group.get("actual_avg_ms"), "winner_observed_avg_ms": None}, "unroutable": False, "rank_error": True, "rank_error_status": status, "rank_error_kind": _backtest_rank_error_kind(data), "rank_error_code": ( (data.get("error") or {}).get("code") if isinstance(data.get("error"), dict) else None ) or data.get("code"), }) continue ranked = rank_cache[route].get("ranked") or [] if not isinstance(ranked, list): ranked = [] winner = ranked[0] if ranked else None backtest = {"winner_provider": None, "winner_model": None, "cost_usd": None, "admitted": len(ranked), "fallbacks": []} unroutable = winner is None winner_latency = None if unroutable: unroutable_requests += requests else: winner_provider = _rank_provider(winner) pin, pout = _rank_prices(winner, route, winner_provider, prices) cf_cost = _counterfactual_cost(tokens_in, tokens_out, pin, pout) if cf_cost is None: unpriced_groups += 1 else: actual_total = round(actual_total + actual_cost, 6) backtest_total = round(backtest_total + cf_cost, 6) winner_latency = provider_latency.get(winner_provider) if winner_provider else None backtest = { "winner_provider": winner_provider, "winner_model": _rank_model(winner), "cost_usd": cf_cost, "admitted": len(ranked), "fallbacks": [_backtest_candidate_summary(r, route, prices) for r in ranked[:3] if isinstance(r, dict)], } groups.append({ "route": route, "requests": requests, "tokens_in": tokens_in, "tokens_out": tokens_out, "actual": {"cost_usd": actual_cost, "providers": providers}, "backtest": backtest, "latency": {"actual_avg_ms": group.get("actual_avg_ms"), "winner_observed_avg_ms": winner_latency}, "unroutable": unroutable, }) if rank_error_groups > 0 and rank_successes == 0 and first_rank_error is not None: status, data = first_rank_error return JSONResponse(status_code=status, content=data) delta = round(backtest_total - actual_total, 6) delta_pct = round((delta / actual_total) * 100.0, 4) if actual_total > 0 else None caveats = [ "Ranked against the current catalog, prices, and breaker state, not the historical state at request time.", "Ranking is treated as request-independent per model family; prompts, tools, cache state, and per-request context are not replayed.", "Costs use current list prices from /x/rank when present, otherwise the host price table.", ] if unpriced_groups: caveats.append("Some admitted winners had no current list price; those groups are excluded from cost totals.") if rank_error_groups: family_word = "family" if rank_error_groups == 1 else "families" caveats.append( f"{rank_error_groups} {family_word} failed to rank; " "their requests are excluded from cost totals.") return JSONResponse(content={ "window": { "timeframe": window_opts["timeframe"], "since_ts": window_opts["since_ts"], "groups_total": int(grouped.get("groups_total") or 0), "groups_shown": int(grouped.get("groups_shown") or 0), "groups_truncated": int(grouped.get("groups_truncated") or 0), "requests_total": int(grouped.get("requests_total") or 0), "requests_covered": int(grouped.get("requests_covered") or 0), "requests_truncated": int(grouped.get("requests_truncated") or 0), }, "totals": { "actual_cost_usd": round(actual_total, 6), "backtest_cost_usd": round(backtest_total, 6), "delta_usd": delta, "delta_pct": delta_pct, "unroutable_requests": unroutable_requests, "rank_error_requests": rank_error_requests, }, "groups": groups, "caveats": caveats, }) @app.post("/dashboard/api/policy/build") async def dashboard_policy_build(request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error return await _router_post_json("/x/policy/build", await request.json()) @app.post("/dashboard/api/policy/templates/{template_id}") async def dashboard_policy_template( template_id: str, request: Request, ) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error return await _router_post_json( f"/x/policy/templates/{template_id}", await request.json(), ) @app.post("/dashboard/api/policy/preview") async def dashboard_policy_preview(request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error return await _router_post_json("/x/rank", await request.json()) @app.post("/dashboard/api/policy/normalize") async def dashboard_policy_normalize(request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error return await _router_post_json("/x/policy/normalize", await request.json()) @app.get("/dashboard/api/fields") async def dashboard_fields(request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error try: r = await _client.get(f"{UPSTREAM}/x/fields", timeout=5.0) return JSONResponse(status_code=r.status_code, content=r.json()) except Exception as exc: # noqa: BLE001 return JSONResponse(status_code=502, content={"error": { "message": str(exc), "type": "router_error", "code": "fields"}}) @app.post("/dashboard/api/policy/test") async def dashboard_policy_test(request: Request) -> Response: """Run the builder's policy live: execute a chat request with this term and a prompt against the router, and record it as an Activity event so its full fallback trace + cost show up under Activity.""" ctx, error = _require_admin_dashboard_auth(request) if error: return error body = await request.json() policy_ir = body.get("policy_ir") prompt = str(body.get("prompt") or "").strip() if not isinstance(policy_ir, list) or not prompt: return JSONResponse(status_code=400, content={"error": { "message": "policy_ir (array) and a non-empty prompt are required", "type": "invalid_request_error", "code": "test_call"}}) payload = {"messages": [{"role": "user", "content": prompt}], "policy_ir": policy_ir, "max_tokens": int(body.get("max_tokens") or 64)} started = time.perf_counter() status = 502 provider = model_family = served_model_id = text = None decision_trace = None tokens_in = tokens_out = tokens_total = 0 cost_usd = None error_type = error_code = error_message = None try: r = await _client.post(f"{UPSTREAM}/v1/chat/completions", json=payload, headers={"x-llm-router-caller": "dashboard-test"}, timeout=45.0) status = r.status_code data = r.json() provider, model_family, served_model_id, decision_trace = _extract_router_metadata(data) xr = data.get("x_router") if isinstance(data, dict) else None if isinstance(xr, dict) and isinstance(xr.get("cost_usd"), (int, float)): cost_usd = float(xr["cost_usd"]) usage = data.get("usage") if isinstance(data, dict) else None if isinstance(usage, dict): tokens_in = int(usage.get("prompt_tokens") or 0) tokens_out = int(usage.get("completion_tokens") or 0) tokens_total = int(usage.get("total_tokens") or (tokens_in + tokens_out)) choices = data.get("choices") if isinstance(data, dict) else None if isinstance(choices, list) and choices: text = ((choices[0] or {}).get("message") or {}).get("content") err = data.get("error") if isinstance(data, dict) else None if isinstance(err, dict): error_type, error_code, error_message = err.get("type"), err.get("code"), err.get("message") except Exception as exc: # noqa: BLE001 error_message = str(exc)[:200] decision_trace = {"attempts": [{"error_kind": "test_error", "message": error_message}]} latency_ms = round((time.perf_counter() - started) * 1000, 1) # record as Activity (in-memory only: key_sha256=None keeps it out of # persistent per-consumer billing history) _record_request(caller="dashboard-test", method="POST", path="/x/policy/test", status=status, latency_ms=latency_ms, provider=provider, model_family=model_family, served_model_id=served_model_id, requested_model="policy_ir", tokens_in=tokens_in, tokens_out=tokens_out, tokens_total=tokens_total, cost_usd=cost_usd, decision_trace=decision_trace, error_type=error_type, error_code=error_code, error_message=_safe_dashboard_message(error_message, limit=300) if error_message else None, key_sha256=None) return JSONResponse(content={ "ok": 200 <= status < 300, "status": status, "text": text, "provider": provider, "model_family": model_family, "served_model_id": served_model_id, "cost_usd": cost_usd, "tokens_total": tokens_total, "tokens_in": tokens_in, "tokens_out": tokens_out, "latency_ms": latency_ms, "decision_trace": decision_trace, "error": error_message}) @app.post("/dashboard/api/flow/normalize") async def dashboard_flow_normalize(request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error return await _router_post_json("/x/flow/normalize", await request.json()) @app.post("/dashboard/api/flow/test") async def dashboard_flow_test(request: Request) -> Response: """Run the Flow Builder's flow live: execute a chat request with this flow_ir and a prompt, and record it as an Activity event so its per-node trace + cost show up under Activity (the flow twin of policy/test).""" ctx, error = _require_admin_dashboard_auth(request) if error: return error body = await request.json() flow_ir = body.get("flow_ir") prompt = str(body.get("prompt") or "").strip() if not isinstance(flow_ir, list) or not prompt: return JSONResponse(status_code=400, content={"error": { "message": "flow_ir (array) and a non-empty prompt are required", "type": "invalid_request_error", "code": "test_call"}}) payload = {"messages": [{"role": "user", "content": prompt}], "flow_ir": flow_ir, "max_tokens": int(body.get("max_tokens") or 64)} started = time.perf_counter() status = 502 provider = model_family = served_model_id = text = None decision_trace = None tokens_in = tokens_out = tokens_total = 0 cost_usd = None error_type = error_code = error_message = None try: r = await _client.post(f"{UPSTREAM}/v1/chat/completions", json=payload, headers={"x-llm-router-caller": "dashboard-test"}, timeout=90.0) status = r.status_code data = r.json() provider, model_family, served_model_id, decision_trace = _extract_router_metadata(data) xr = data.get("x_router") if isinstance(data, dict) else None if isinstance(xr, dict) and isinstance(xr.get("cost_usd"), (int, float)): cost_usd = float(xr["cost_usd"]) usage = data.get("usage") if isinstance(data, dict) else None if isinstance(usage, dict): tokens_in = int(usage.get("prompt_tokens") or 0) tokens_out = int(usage.get("completion_tokens") or 0) tokens_total = int(usage.get("total_tokens") or (tokens_in + tokens_out)) choices = data.get("choices") if isinstance(data, dict) else None if isinstance(choices, list) and choices: text = ((choices[0] or {}).get("message") or {}).get("content") err = data.get("error") if isinstance(data, dict) else None if isinstance(err, dict): error_type, error_code, error_message = err.get("type"), err.get("code"), err.get("message") except Exception as exc: # noqa: BLE001 error_message = str(exc)[:200] decision_trace = {"attempts": [{"error_kind": "test_error", "message": error_message}]} latency_ms = round((time.perf_counter() - started) * 1000, 1) _record_request(caller="dashboard-test", method="POST", path="/x/flow/test", status=status, latency_ms=latency_ms, provider=provider, model_family=model_family, served_model_id=served_model_id, requested_model="flow_ir", tokens_in=tokens_in, tokens_out=tokens_out, tokens_total=tokens_total, cost_usd=cost_usd, decision_trace=decision_trace, error_type=error_type, error_code=error_code, error_message=_safe_dashboard_message(error_message, limit=300) if error_message else None, key_sha256=None) return JSONResponse(content={ "ok": 200 <= status < 300, "status": status, "text": text, "provider": provider, "model_family": model_family, "served_model_id": served_model_id, "cost_usd": cost_usd, "tokens_total": tokens_total, "tokens_in": tokens_in, "tokens_out": tokens_out, "latency_ms": latency_ms, "decision_trace": decision_trace, "error": error_message}) @app.get("/dashboard/api/keys") async def dashboard_keys(request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error caller = str(ctx.get("viewer")) return JSONResponse(content={"keys": _consumer_key_rows(), "consumers": _consumers()}) @app.get("/dashboard/api/logins") async def dashboard_logins(request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error timeframe = _dashboard_timeframe(request.query_params.get("timeframe") or request.query_params.get("window")) consumer = (request.query_params.get("consumer") or "").strip() or None return JSONResponse(content=_login_connections_snapshot(timeframe=timeframe, consumer=consumer, viewer_role="admin")) @app.get("/dashboard/api/provider-keys") async def dashboard_provider_keys(request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error timeframe = _dashboard_timeframe(request.query_params.get("timeframe") or request.query_params.get("window")) snap = _provider_credentials_snapshot(timeframe=timeframe, viewer_role="admin") await _attach_antseed_wallet(snap) return JSONResponse(content=snap) @app.post("/dashboard/api/key-usage") async def dashboard_key_usage(request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error caller = str(ctx.get("viewer")) try: data = await request.json() if not isinstance(data, dict): raise ValueError("JSON body must be an object") api_key = str(data.get("api_key") or "").strip() if not api_key: raise ValueError("api_key is required") options = _usage_query_options(data) except ValueError as exc: return JSONResponse(status_code=400, content={"error": {"message": str(exc), "type": "invalid_request_error", "code": "invalid_key_usage_request"}}) except Exception: return JSONResponse(status_code=400, content={"error": {"message": "invalid JSON body", "type": "invalid_request_error", "code": "invalid_json"}}) digest = hashlib.sha256(api_key.encode()).hexdigest() owner = CALLER_KEYS.get(api_key) or CALLER_KEY_HASHES.get(digest) with _stats_lock: has_usage = int(_stats["by_key_sha256"].get(digest, {}).get("requests") or 0) > 0 owner = owner or _stats["key_owner"].get(digest) if not owner and not has_usage: rows, persistent = _usage_events_for_key(digest, owner) has_usage = bool(rows) if not owner and not has_usage: return JSONResponse(status_code=404, content={"error": {"message": "no configured key or recorded usage found for api_key", "type": "not_found", "code": "key_usage_not_found"}}) return JSONResponse(content=_key_usage_snapshot(viewer=caller, key_sha256=digest, caller=owner, options=options)) @app.get("/v1/usage") @app.get("/api/usage") async def key_usage(request: Request) -> Response: token = _extract_token(request) auth = await _caller_auth_async(token) if not auth.get("ok"): code = auth.get("error_code") or "caller_auth" status_code = 403 if code in {"caller_inactive", "caller_key_revoked", "caller_key_expired"} else 401 messages = { "caller_auth": "unauthorized caller", "caller_inactive": "caller is inactive", "caller_key_revoked": "caller key is revoked", "caller_key_expired": "caller key is expired", } return JSONResponse(status_code=status_code, content={"error": {"message": messages.get(code, "caller not authorized"), "type": "auth_error", "code": code}}) caller = str(auth.get("caller")) try: options = _usage_query_options(request.query_params) except ValueError as exc: return JSONResponse(status_code=400, content={"error": {"message": str(exc), "type": "invalid_request_error", "code": "invalid_usage_window"}}) if (limited := await _consumer_rate_limited(caller, auth)) is not None: return limited # Postgres reads + per-row pricing: off the event loop. snap = await asyncio.to_thread(_key_usage_snapshot, viewer=f"consumer:{caller}", key_sha256=str(auth.get("digest")), caller=caller, options=options) return JSONResponse(content=snap) # Opaque client-chosen session ids (opencode `ses_...`, UUIDs, `tenant:x:y`); # never a bare dot segment, which would re-target the upstream path. _SESSION_ID_RE = re.compile(r"[A-Za-z0-9_:-][A-Za-z0-9_.:-]{0,127}") async def _consumer_rate_limited(caller: str, auth: dict[str, Any]) -> JSONResponse | None: """The caller's request rate limit, shared with proxied traffic, for the consumer read endpoints (/v1/usage, /v1/session).""" result = await asyncio.to_thread(_rate_ok, caller, auth.get("meta") or {}) allowed, store_ok, retry_after_s = result if isinstance(result, tuple) else (bool(result), True, 0.0) if not store_ok: _metric_store_error("rate_limit") return JSONResponse(status_code=503, content={"error": { "message": "caller rate-limit state is temporarily unavailable", "type": "server_error", "code": "caller_rate_limit_unavailable"}}) if not allowed: return JSONResponse(status_code=429, headers={"Retry-After": str(max(1, math.ceil(retry_after_s)))}, content={"error": {"message": "caller rate limit exceeded", "type": "rate_limit_error", "code": "caller_rate_limit"}}) return None @app.get("/v1/session/{sid}") async def session_view(sid: str, request: Request) -> Response: """Consumer-facing per-session economics: accumulated cost / tokens / cache, plus `warm` — the routes (family / provider / served_by, i.e. the real peer or backend behind a marketplace) currently holding the session's prompt-cache prefix. Authed by the caller's consumer key; `sid` is the opaque id the client itself sends as X-Unhardcoded-Session. Lets a harness (e.g. the opencode plugin) show live router economics without operator access to /x/*.""" token = _extract_token(request) auth = await _caller_auth_async(token) if not auth.get("ok"): code = auth.get("error_code") or "caller_auth" status_code = 403 if code in {"caller_inactive", "caller_key_revoked", "caller_key_expired"} else 401 return JSONResponse(status_code=status_code, content={"error": {"message": "caller not authorized", "type": "auth_error", "code": code}}) assert _client is not None # Forward the authed consumer key so the upstream meter scopes the read to # the session's OWNER (bound when the sid was first metered). A consumer must # not read another consumer's session economics / warm peers; the upstream # answers 404 (not 403) for a sid this caller does not own, so the endpoint # never confirms that someone else's sid exists. caller = str(auth.get("caller") or "") if not _SESSION_ID_RE.fullmatch(sid): return JSONResponse(status_code=400, content={"error": {"message": "invalid session id", "type": "invalid_request_error", "code": "invalid_session_id"}}) if (limited := await _consumer_rate_limited(caller, auth)) is not None: return limited try: r = await _client.get(f"{UPSTREAM}/x/session/{quote(sid, safe='')}", timeout=5.0, headers=_router_admin_headers({"x-llm-router-caller": caller})) return JSONResponse(status_code=r.status_code, content=r.json()) except Exception as exc: return JSONResponse(status_code=502, content={"error": {"message": f"upstream: {exc}", "type": "api_error", "code": "upstream_error"}}) @app.post("/dashboard/api/consumers/{consumer}") async def dashboard_update_consumer(consumer: str, request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error caller = str(ctx.get("viewer")) try: consumer = _safe_consumer_name(consumer) data = await request.json() if not isinstance(data, dict): raise ValueError("JSON body must be an object") except ValueError as exc: return JSONResponse(status_code=400, content={"error": {"message": str(exc), "type": "invalid_request_error", "code": "invalid_consumer"}}) except Exception: return JSONResponse(status_code=400, content={"error": {"message": "invalid JSON body", "type": "invalid_request_error", "code": "invalid_json"}}) records = _writable_consumer_records() if records is None: return _consumer_store_unavailable() meta = records.get(consumer, _normalize_consumer_record(consumer, {})) if "status" in data: status = str(data.get("status") or "").strip().lower() if status not in {"active", "inactive"}: return JSONResponse(status_code=400, content={"error": {"message": "status must be active or inactive", "type": "invalid_request_error", "code": "invalid_status"}}) meta["status"] = status if "allowed_routes" in data: meta["allowed_routes"] = _clean_route_list(data.get("allowed_routes")) if "rate_per_min" in data: meta["rate_per_min"] = _optional_int(data.get("rate_per_min"), min_value=1) if "burst" in data: meta["burst"] = _optional_int(data.get("burst"), min_value=1) meta["updated_at"] = int(time.time()) records[consumer] = meta if not _write_issued_consumer_records(records): return JSONResponse(status_code=500, content={"ok": False, "error": "failed to persist consumer settings"}) _log({"event": "dashboard_consumer_updated", "consumer": consumer, "viewer": caller, "status": meta.get("status")}) return JSONResponse(content={"ok": True, "consumer": consumer, "settings": _normalize_consumer_record(consumer, meta)}) @app.post("/dashboard/api/keys/revoke") async def dashboard_revoke_key(request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error caller = str(ctx.get("viewer")) try: data = await request.json() if not isinstance(data, dict): raise ValueError("JSON body must be an object") consumer = _safe_consumer_name(str(data.get("consumer", ""))) prefix = str(data.get("sha256_prefix") or "").strip().lower() if not re.fullmatch(r"[a-f0-9]{8,64}", prefix): raise ValueError("sha256_prefix must be 8-64 lowercase hex chars") except ValueError as exc: return JSONResponse(status_code=400, content={"error": {"message": str(exc), "type": "invalid_request_error", "code": "invalid_key"}}) except Exception: return JSONResponse(status_code=400, content={"error": {"message": "invalid JSON body", "type": "invalid_request_error", "code": "invalid_json"}}) records = _writable_consumer_records() if records is None: return _consumer_store_unavailable() meta = records.get(consumer, _normalize_consumer_record(consumer, {})) now = int(time.time()) found = False for key in meta.get("keys") or []: if str(key.get("sha256_prefix") or "").startswith(prefix) or prefix.startswith(str(key.get("sha256_prefix") or "")): key["status"] = "revoked" key["revoked_at"] = now found = True new_hashes = {digest: owner for digest, owner in CALLER_KEY_HASHES.items() if not (owner == consumer and digest.startswith(prefix))} removed = len(CALLER_KEY_HASHES) - len(new_hashes) if removed: found = True new_plaintext = {token: owner for token, owner in CALLER_KEYS.items() if not (owner == consumer and hashlib.sha256(token.encode()).hexdigest().startswith(prefix))} removed_plaintext = len(CALLER_KEYS) - len(new_plaintext) if removed_plaintext: found = True if not found: return JSONResponse(status_code=404, content={"error": {"message": "key prefix not found for consumer", "type": "not_found", "code": "key_not_found"}}) meta["updated_at"] = now records[consumer] = meta if not _write_issued_consumer_records( records, delete_key_digests=[(consumer, prefix)]): return JSONResponse(status_code=500, content={"ok": False, "error": "failed to persist key revocation"}) # Publish the local/PVC mirror only after the shared authority commits. A # failed SQL write must never revoke on one replica while others accept it. if removed: _publish_local_consumer_map( "CALLER_KEYS_SHA256_JSON", new_hashes, CALLER_KEY_HASHES) if removed_plaintext: _publish_local_consumer_map( "CALLER_KEYS_JSON", new_plaintext, CALLER_KEYS) _log({"event": "dashboard_key_revoked", "consumer": consumer, "viewer": caller, "sha256_prefix": prefix, "removed_hashes": removed, "removed_plaintext": removed_plaintext}) return JSONResponse(content={"ok": True, "consumer": consumer, "sha256_prefix": prefix, "removed_hashes": removed, "removed_plaintext": removed_plaintext}) @app.post("/dashboard/api/provider-keys/add") async def dashboard_add_provider(request: Request) -> Response: """Add a provider + its API key from the dashboard. Persists the key to .env.secrets (auth_env indirection) and the provider definition to the host store (provider_overlays), then hot-applies it via the router's /x/providers so it serves without a restart. Admin-only, like key reveal.""" caller, error = _require_admin_dashboard_caller(request) if error: return error try: body = await request.json() except Exception: return JSONResponse(status_code=400, content={"error": {"message": "invalid JSON body", "type": "invalid_request", "code": "provider_add"}}) pid = str(body.get("id") or "").strip().lower() auth_env = str(body.get("auth_env") or "").strip() key = str(body.get("key") or "").strip() entry = { "base_url": str(body.get("base_url") or "").strip(), "api_kind": "openai_compatible", "tier": str(body.get("tier") or "partner").strip() or "partner", "auth_env": auth_env, "served_models": [sm for sm in (body.get("served_models") or []) if isinstance(sm, dict)], "added_at": int(time.time()), } try: from provider_overlay import load_overlay, save_overlay, validate_entry catalog = _load_policy_config() # includes existing overlay entries errors = validate_entry(pid, entry, catalog, key_supplied=bool(key)) if not key: errors.append("key is required") elif (env_error := _provider_env_error(auth_env, key)) is not None: errors.append(env_error) if errors: return JSONResponse(status_code=400, content={"error": {"message": "; ".join(errors), "type": "invalid_request", "code": "provider_add"}}) _upsert_env_line(Path(DASHBOARD_KEY_ENV_PATH), auth_env, key) overlay = load_overlay() overlay.setdefault("providers", {})[pid] = entry if not save_overlay(overlay): return JSONResponse(status_code=500, content={"error": {"message": "failed to persist provider overlay", "type": "provider_add_error", "code": "provider_add"}}) except Exception as exc: return JSONResponse(status_code=500, content={"error": {"message": str(exc), "type": "provider_add_error", "code": "provider_add"}}) # hot-apply on the running router; on failure the provider still applies # at the next router restart (overlay + env are already persisted) applied_live = False apply_error = None try: if _client is not None: r = await _client.post(f"{UPSTREAM}/x/providers", json={ "id": pid, **{k: v for k, v in entry.items() if k != "added_at"}, "key": key}, timeout=10.0, headers=_router_admin_headers()) applied_live = r.status_code == 200 if not applied_live: apply_error = f"router /x/providers returned {r.status_code}" except Exception as exc: apply_error = str(exc) _log({"event": "dashboard_provider_added", "provider": pid, "viewer": caller, "auth_env": auth_env, "applied_live": applied_live}) return JSONResponse(content={ "ok": True, "provider": pid, "applied_live": applied_live, "note": None if applied_live else f"saved, but live apply failed ({apply_error}); the provider will load on the next router restart"}) @app.post("/dashboard/api/provider-keys/update") async def dashboard_update_provider_key(request: Request) -> Response: """Replace the API key of an EXISTING provider (heurist/ionet/openrouter, or any operator-added one). Persists the new key to .env.secrets — the PVC source of truth that survives restarts — and hot-applies it via the router's /x/provider-key so it serves without a restart. Admin-only, like add/reveal.""" caller, error = _require_admin_dashboard_caller(request) if error: return error try: body = await request.json() except Exception: return JSONResponse(status_code=400, content={"error": {"message": "invalid JSON body", "type": "invalid_request", "code": "provider_update"}}) pid = str(body.get("provider") or body.get("id") or "").strip() key = str(body.get("key") or "").strip() if not pid: return JSONResponse(status_code=400, content={"error": {"message": "provider is required", "type": "invalid_request", "code": "provider_update"}}) if not key: return JSONResponse(status_code=400, content={"error": {"message": "key is required", "type": "invalid_request", "code": "provider_update"}}) try: cfg = _load_policy_config() except Exception as exc: return JSONResponse(status_code=500, content={"error": {"message": str(exc), "type": "provider_update_error", "code": "provider_update"}}) provider = (cfg.get("providers") or {}).get(pid) if not isinstance(provider, dict): return JSONResponse(status_code=404, content={"error": {"message": f"provider {pid!r} not found", "type": "not_found", "code": "provider_not_found"}}) auth_env = str(provider.get("auth_env") or "").strip() if not auth_env: return JSONResponse(status_code=400, content={"error": {"message": f"provider {pid!r} has no auth_env (e.g. oauth/codex); use the Codex account flow instead", "type": "invalid_request", "code": "provider_no_auth_env"}}) if (env_error := _provider_env_error(auth_env, key)) is not None: return JSONResponse(status_code=400, content={"error": {"message": env_error, "type": "invalid_request", "code": "provider_update"}}) try: _upsert_env_line(Path(DASHBOARD_KEY_ENV_PATH), auth_env, key) except Exception as exc: return JSONResponse(status_code=500, content={"error": {"message": str(exc), "type": "provider_update_error", "code": "provider_update"}}) # hot-apply on the running router; on failure the key still loads from # .env.secrets at the next router restart (now that it's persisted) applied_live = False apply_error = None try: if _client is not None: r = await _client.post(f"{UPSTREAM}/x/provider-key", json={ "provider": pid, "auth_env": auth_env, "key": key}, timeout=10.0, headers=_router_admin_headers()) applied_live = r.status_code == 200 if not applied_live: apply_error = f"router /x/provider-key returned {r.status_code}" except Exception as exc: apply_error = str(exc) _log({"event": "dashboard_provider_key_updated", "provider": pid, "viewer": caller, "auth_env": auth_env, "applied_live": applied_live}) return JSONResponse(content={ "ok": True, "provider": pid, "applied_live": applied_live, "note": None if applied_live else f"saved, but live apply failed ({apply_error}); the key will load on the next router restart"}) async def _wallet_proxy(request: Request, op: str, *, body: dict | None = None, timeout: float = 135.0) -> Response: """Admin-only proxy to the router's /x/wallet/* — which runs the AntSeed buyer deposit/withdraw/refresh/reclaim on the sidecar and returns the refreshed wallet. Lets the operator fund and reclaim the hot-wallet from the catalog instead of `kubectl exec`.""" caller, error = _require_admin_dashboard_caller(request) if error: return error if _client is None: return JSONResponse(status_code=502, content={"error": { "message": "router client unavailable", "type": "wallet_error", "code": "wallet"}}) try: r = await _client.post(f"{UPSTREAM}/x/wallet/{op}", json=(body or {}), timeout=timeout, headers=_router_admin_headers()) except Exception as exc: return JSONResponse(status_code=502, content={"error": { "message": f"router /x/wallet/{op} unreachable: {exc}", "type": "wallet_error", "code": "wallet"}}) _log({"event": "dashboard_wallet_op", "op": op, "viewer": caller, "status": r.status_code, "amount": (body or {}).get("amount")}) try: return JSONResponse(status_code=r.status_code, content=r.json()) except Exception: return JSONResponse(status_code=502, content={"error": { "message": (r.text or "")[:300], "type": "wallet_error", "code": "wallet"}}) @app.post("/dashboard/api/wallet/deposit") async def dashboard_wallet_deposit(request: Request) -> Response: try: body = await request.json() except Exception: body = {} return await _wallet_proxy(request, "deposit", body={"amount": str(body.get("amount", "")).strip()}) @app.post("/dashboard/api/wallet/withdraw") async def dashboard_wallet_withdraw(request: Request) -> Response: try: body = await request.json() except Exception: body = {} return await _wallet_proxy(request, "withdraw", body={"amount": str(body.get("amount", "")).strip()}) @app.post("/dashboard/api/wallet/refresh") async def dashboard_wallet_refresh(request: Request) -> Response: return await _wallet_proxy(request, "refresh") @app.post("/dashboard/api/wallet/reclaim/scan") async def dashboard_wallet_reclaim_scan(request: Request) -> Response: # Read-only: enumerate channels + on-chain reclaimable USDC + operator status. return await _wallet_proxy(request, "reclaim/scan", timeout=100.0) @app.post("/dashboard/api/wallet/reclaim/set-operator") async def dashboard_wallet_reclaim_set_operator(request: Request) -> Response: # On-chain one-time: assign the buyer wallet as its own deposits operator # (unblocks requestClose/withdraw). Moves no funds. return await _wallet_proxy(request, "reclaim/set-operator", timeout=260.0) @app.post("/dashboard/api/wallet/reclaim/request-close") async def dashboard_wallet_reclaim_request_close(request: Request) -> Response: # On-chain: start the ~15-min close challenge on idle channels. return await _wallet_proxy(request, "reclaim/request-close", timeout=260.0) @app.post("/dashboard/api/wallet/reclaim/withdraw") async def dashboard_wallet_reclaim_withdraw(request: Request) -> Response: # On-chain: pull funds from channels whose challenge window has elapsed. return await _wallet_proxy(request, "reclaim/withdraw", timeout=260.0) @app.get("/dashboard/api/wallet") async def dashboard_wallet_get(request: Request) -> Response: """Read-only wallet balance for the config panel — no on-chain tx.""" caller, error = _require_admin_dashboard_caller(request) if error: return error if _client is None: return JSONResponse(status_code=502, content={"error": { "message": "router client unavailable", "type": "wallet_error", "code": "wallet"}}) try: r = await _client.get(f"{UPSTREAM}/x/wallet", timeout=10.0, headers=_router_admin_headers()) return JSONResponse(status_code=r.status_code, content=r.json()) except Exception as exc: return JSONResponse(status_code=502, content={"error": { "message": f"router /x/wallet unreachable: {exc}", "type": "wallet_error", "code": "wallet"}}) @app.get("/dashboard/api/provider-keys/reveal") async def dashboard_reveal_provider_key(request: Request) -> Response: caller, error = _require_admin_dashboard_caller(request) if error: return error name = str(request.query_params.get("provider") or "").strip() kind, value = _provider_raw_credential(name) _log({"event": "dashboard_provider_key_revealed", "provider": name, "viewer": caller, "kind": kind, "found": bool(value)}) if not value: return JSONResponse(status_code=404, content={"error": { "message": f"no recoverable credential for provider {name!r}", "type": "not_found", "code": "provider_key_reveal"}}) return JSONResponse(content={ "provider": name, "kind": kind, "value": value, "fingerprint": hashlib.sha256(value.encode()).hexdigest()[:12]}) def _codex_store(): from codex_auth import CodexAuthStore return CodexAuthStore(CODEX_ACCOUNTS_DIR, legacy_path=CODEX_AUTH_PATH) def _codex_provider_id() -> str | None: """The provider id of the codex (openai_codex) backend, from the catalog.""" try: cfg = _load_policy_config() except Exception: return None for pid, p in (cfg.get("providers") or {}).items(): if isinstance(p, dict) and p.get("api_kind") == "openai_codex": return pid return None async def _codex_activity() -> dict[str, Any]: """Codex provider activity for the dashboard: request/error counts (from the ingress stats) plus the live quota + scarcity-price state (from the router's /x/runtime — the codex source's quota_window and the imputed ranking price). So the Codex panel shows what the other providers' rows show, and more.""" pid = _codex_provider_id() or "openai" with _stats_lock: snap = _counter_snapshot(_stats["by_provider"].get(pid) or _counter()) rt = await _fetch_router_runtime() or {} bal = (rt.get("balances") or {}).get(pid) or {} detail = bal.get("detail") or {} used = bal.get("value") price_in = None for k, m in (rt.get("ema_metrics") or {}).items(): if isinstance(m, dict) and k.startswith(pid + "|") and m.get("price_in") is not None: price_in = m["price_in"] break return { "provider": pid, "requests": snap.get("requests", 0), "errors": snap.get("errors", 0), "error_rate": snap.get("error_rate", 0.0), "tokens_total": snap.get("tokens_total", 0), "last_seen": snap.get("last_seen"), "used_percent": round(used * 100, 1) if isinstance(used, (int, float)) else None, "recent_429": detail.get("recent_429_count"), "scarcity_price_in": price_in, "events": detail.get("events"), } async def _reload_codex_router() -> tuple[bool, str | None]: """Ask the router to re-scan the Codex accounts dir so a dashboard change goes live. On failure the change still loads at the next router restart.""" try: if _client is not None: if CODEX_BROKER_URL: if not CODEX_BROKER_TOKEN: return False, "CODEX_BROKER_TOKEN is not configured" url = f"{CODEX_BROKER_URL}/v1/reload" headers = {"authorization": f"Bearer {CODEX_BROKER_TOKEN}"} else: url = f"{UPSTREAM}/x/codex/reload" headers = _router_admin_headers() r = await _client.post(url, headers=headers, timeout=10.0) if r.status_code == 200: return True, None return False, f"Codex reload returned {r.status_code}" except Exception as exc: return False, str(exc) return False, "router client unavailable" @app.get("/dashboard/api/codex/accounts") async def dashboard_list_codex_accounts(request: Request) -> Response: """List the Codex (ChatGPT-subscription) accounts on the PVC — name, account_id and a token fingerprint, never the raw token. Admin-only.""" caller, error = _require_admin_dashboard_caller(request) if error: return error try: store = _codex_store() accounts = store.list_accounts() selection = store.selection() active = store.active_label() # an account name, or "balanced" except Exception as exc: return JSONResponse(status_code=500, content={"error": {"message": str(exc), "type": "codex_list_error", "code": "codex_accounts"}}) for a in accounts: a["active"] = (a["name"] == active) return JSONResponse(content={"accounts": accounts, "active": active, "selection": selection, "activity": await _codex_activity()}) @app.post("/dashboard/api/codex/select") async def dashboard_select_codex_account(request: Request) -> Response: """Choose which Codex account serves traffic: a specific account, or 'balanced' (round-robin across all). Persisted on the PVC (shared with the router via the accounts dir), so it takes effect without a restart. Admin-only.""" caller, error = _require_admin_dashboard_caller(request) if error: return error try: body = await request.json() except Exception: return JSONResponse(status_code=400, content={"error": {"message": "invalid JSON body", "type": "invalid_request", "code": "codex_select"}}) mode = str(body.get("mode") or "").strip() account = body.get("account") try: selection = _codex_store().set_selection(mode, account) except ValueError as exc: return JSONResponse(status_code=400, content={"error": {"message": str(exc), "type": "invalid_request", "code": "codex_select"}}) except Exception as exc: return JSONResponse(status_code=500, content={"error": {"message": str(exc), "type": "codex_select_error", "code": "codex_select"}}) # The router reads the selection live from the shared accounts dir; a reload # is only needed for new accounts, but trigger one anyway so it's immediate. applied_live, _ = await _reload_codex_router() _log({"event": "dashboard_codex_account_selected", "selection": selection, "viewer": caller, "applied_live": applied_live}) return JSONResponse(content={"ok": True, "selection": selection}) @app.post("/dashboard/api/codex/accounts") async def dashboard_add_codex_account(request: Request) -> Response: """Add/replace a Codex account by pasting its auth.json (the output of `codex login`). Persisted to the PVC and hot-applied on the router. Admin-only, like provider keys.""" caller, error = _require_admin_dashboard_caller(request) if error: return error try: body = await request.json() except Exception: return JSONResponse(status_code=400, content={"error": {"message": "invalid JSON body", "type": "invalid_request", "code": "codex_add"}}) name = str(body.get("name") or "").strip() auth_json = body.get("auth_json") if isinstance(auth_json, str): try: auth_json = json.loads(auth_json) except Exception: return JSONResponse(status_code=400, content={"error": {"message": "auth_json is not valid JSON", "type": "invalid_request", "code": "codex_add"}}) if not isinstance(auth_json, dict): return JSONResponse(status_code=400, content={"error": {"message": "auth_json (object or JSON string) is required", "type": "invalid_request", "code": "codex_add"}}) try: slug = _codex_store().add_account(name, auth_json) except ValueError as exc: return JSONResponse(status_code=400, content={"error": {"message": str(exc), "type": "invalid_request", "code": "codex_add"}}) except Exception as exc: return JSONResponse(status_code=500, content={"error": {"message": str(exc), "type": "codex_add_error", "code": "codex_add"}}) applied_live, apply_error = await _reload_codex_router() _log({"event": "dashboard_codex_account_added", "account": slug, "viewer": caller, "applied_live": applied_live}) return JSONResponse(content={ "ok": True, "account": slug, "applied_live": applied_live, "note": None if applied_live else f"saved, but live reload failed ({apply_error}); the account will load on the next router restart"}) @app.delete("/dashboard/api/codex/accounts/{name}") async def dashboard_delete_codex_account(request: Request, name: str) -> Response: """Remove a Codex account from the PVC and hot-reload the router. Admin-only.""" caller, error = _require_admin_dashboard_caller(request) if error: return error try: existed = _codex_store().delete_account(name) except ValueError as exc: return JSONResponse(status_code=400, content={"error": {"message": str(exc), "type": "invalid_request", "code": "codex_delete"}}) except Exception as exc: return JSONResponse(status_code=500, content={"error": {"message": str(exc), "type": "codex_delete_error", "code": "codex_delete"}}) if not existed: return JSONResponse(status_code=404, content={"error": {"message": f"codex account {name!r} not found", "type": "not_found", "code": "codex_account_not_found"}}) applied_live, apply_error = await _reload_codex_router() _log({"event": "dashboard_codex_account_deleted", "account": name, "viewer": caller, "applied_live": applied_live}) return JSONResponse(content={"ok": True, "account": name, "applied_live": applied_live}) # ---- Codex onboarding invites (server-side OAuth device flow) -------------- # Spec: docs/superpowers/specs/2026-07-22-codex-oauth-invite-design.md def _invite_store(): from codex_invites import CodexInviteStore return CodexInviteStore(CODEX_ACCOUNTS_DIR) def _request_origin(request: Request) -> str: """Public origin for links we hand out, honoring the ingress's X-Forwarded-* headers; falls back to the request's own scheme/host.""" proto = (request.headers.get("x-forwarded-proto") or request.url.scheme or "http") host = (request.headers.get("x-forwarded-host") or request.headers.get("host") or request.url.netloc) return f"{proto.split(',')[0].strip()}://{host.split(',')[0].strip()}" def _invite_view(inv: dict, origin: str) -> dict: return {"name": inv["name"], "status": inv.get("status"), "url": f"{origin}/codex/onboard/{inv['token']}", "created_at": inv.get("created_at"), "expires_at": inv.get("expires_at"), "used_at": inv.get("used_at")} @app.post("/dashboard/api/codex/invites") async def dashboard_create_codex_invite(request: Request) -> Response: """Mint a single-use onboarding link that lets a teammate OAuth their ChatGPT account into the named Codex slot. Admin-only.""" caller, error = _require_admin_dashboard_caller(request) if error: return error try: body = await request.json() except Exception: return JSONResponse(status_code=400, content={"error": {"message": "invalid JSON body", "type": "invalid_request", "code": "codex_invite"}}) try: inv = _invite_store().create(str(body.get("name") or "")) except ValueError as exc: return JSONResponse(status_code=400, content={"error": {"message": str(exc), "type": "invalid_request", "code": "codex_invite"}}) _log({"event": "dashboard_codex_invite_created", "account": inv["name"], "viewer": caller}) return JSONResponse(content={"ok": True, **_invite_view({**inv, "status": "pending"}, _request_origin(request))}) @app.get("/dashboard/api/codex/invites") async def dashboard_list_codex_invites(request: Request) -> Response: caller, error = _require_admin_dashboard_caller(request) if error: return error origin = _request_origin(request) return JSONResponse(content={"invites": [_invite_view(i, origin) for i in _invite_store().list()]}) @app.delete("/dashboard/api/codex/invites/{token}") async def dashboard_revoke_codex_invite(request: Request, token: str) -> Response: caller, error = _require_admin_dashboard_caller(request) if error: return error if not _invite_store().revoke(token): return JSONResponse(status_code=404, content={"error": {"message": "invite not found", "type": "not_found", "code": "codex_invite_not_found"}}) _log({"event": "dashboard_codex_invite_revoked", "viewer": caller}) return JSONResponse(content={"ok": True}) def _live_invite(token: str): """(store, invite, status). Unknown and expired links both present to the visitor as a dead link (no oracle distinguishing them).""" store = _invite_store() inv = store.get(token) if inv is None: return store, None, "missing" return store, inv, store.status_of(inv) @app.get("/codex/onboard/{token}") async def codex_onboard_page(token: str) -> Response: store, inv, status = _live_invite(token) if inv is None or status == "expired": return HTMLResponse(_onboard_dead_html(), status_code=404) return HTMLResponse(_onboard_html(inv["name"], connected=(status == "used"))) @app.post("/codex/onboard/{token}/start") async def codex_onboard_start(token: str) -> Response: import codex_auth store, inv, status = _live_invite(token) if inv is None or status == "expired": return JSONResponse(status_code=404, content={"error": {"message": "invite link expired", "type": "not_found", "code": "codex_onboard"}}) if status == "used": return JSONResponse(status_code=409, content={"error": {"message": "invite already used", "type": "invalid_request", "code": "codex_onboard_used"}}) try: device = await asyncio.to_thread(codex_auth.device_usercode_request) except codex_auth.DeviceAuthError as exc: return JSONResponse(status_code=502, content={"error": {"message": str(exc), "type": "device_auth_error", "code": "codex_onboard_start"}}) store.set_device(token, device) _log({"event": "codex_invite_signin_started", "account": inv["name"]}) return JSONResponse(content={"user_code": device["user_code"], "verification_url": codex_auth.DEVICE_VERIFY_URL, "interval": device["interval"]}) @app.get("/codex/onboard/{token}/status") async def codex_onboard_status(token: str) -> Response: import codex_auth store, inv, status = _live_invite(token) if inv is None or status == "expired": # Same shape as an unknown token: no oracle distinguishing the two. return JSONResponse(status_code=404, content={"error": {"message": "invite link expired", "type": "not_found", "code": "codex_onboard"}}) if status == "used": return JSONResponse(content={"status": "connected", "name": inv["name"]}) if status == "pending": if inv.get("device_auth_id"): store.clear_device(token) # device code timed out; allow a fresh start return JSONResponse(content={"status": "pending"}) # status == "awaiting": at most one upstream poll per interval if not store.due_for_poll(token): return JSONResponse(content={"status": "awaiting"}) try: result = await asyncio.to_thread( codex_auth.device_token_poll, inv["device_auth_id"], inv["user_code"]) if result is None: return JSONResponse(content={"status": "awaiting"}) auth_json = await asyncio.to_thread( codex_auth.device_code_exchange, result["authorization_code"], result["code_verifier"]) except codex_auth.DeviceAuthError as exc: store.clear_device(token) _log({"event": "codex_invite_device_error", "account": inv["name"], "error": str(exc)}) return JSONResponse(content={"status": "error", "message": str(exc)}) slug = _codex_store().add_account(inv["name"], auth_json) applied_live, _ = await _reload_codex_router() store.mark_used(token) _log({"event": "codex_invite_connected", "account": slug, "applied_live": applied_live}) return JSONResponse(content={"status": "connected", "name": slug, "applied_live": applied_live}) def _onboard_dead_html() -> str: return """ Link expired

This link is no longer valid

The invite has expired or was already used. Ask the person who sent it for a new link.

""" def _onboard_html(name: str, connected: bool = False) -> str: import html as _html safe = _html.escape(name) idle_cls = "hidden" if connected else "" done_cls = "" if connected else "hidden" return f""" Connect ChatGPT — {safe}

Connect your ChatGPT account

This links your ChatGPT subscription to the unhardcoded router as account {safe}. Only continue if a person you trust sent you this link.

You'll sign in on openai.com and enter a one-time code. OpenAI's page warns about codes given to you by websites — that warning refers to this flow; continue only because your operator sent you this link, otherwise cancel.

✓ Connected as {safe}. You can close this page.

""" def _cost_accuracy_rows(routes, ema, _mult_of, *, min_calls=20, threshold=0.15): """Per-provider measured-vs-list cost, joining the ledger aggregate (`routes`, from host_store.cost_by_route) with the live ranked prices (`ema`, "|" -> raw {price_in, price_out}). The fictitious price_multiplier is applied at selection time and is not persisted here, so the comparison is measured-spend vs advertised-list directly. A route with no known or non-positive price is skipped (nothing to compare). Pure (no IO) so the join/deviation logic is unit-tested directly.""" roll: dict[str, dict] = {} for r in routes: m = ema.get(f"{r['provider']}|{r['family']}") if not isinstance(m, dict): continue try: list_in = float(m.get("price_in")) list_out = float(m.get("price_out")) except (TypeError, ValueError): continue if list_in < 0 or list_out < 0: # unpriced sentinel / +inf continue uncached = max(0, r["tokens_in"] - r["tokens_cached"]) expected = (uncached * list_in + r["tokens_cached"] * list_in * _CACHE_READ_FACTOR + r["tokens_out"] * list_out) / 1e6 agg = roll.setdefault(r["provider"], { "provider": r["provider"], "calls": 0, "tokens_total": 0, "measured_usd": 0.0, "expected_usd": 0.0, "n_reported": 0}) agg["calls"] += r["calls"] agg["tokens_total"] += r["tokens_in"] + r["tokens_out"] agg["measured_usd"] += r["cost_usd"] agg["expected_usd"] += expected agg["n_reported"] += r.get("n_reported", 0) rows = [] for a in roll.values(): if a["expected_usd"] <= 0 or a["tokens_total"] <= 0: continue mtok = a["tokens_total"] / 1e6 deviation = a["measured_usd"] / a["expected_usd"] # 'reported' = the provider's own authoritative cost (independent signal); # 'derived' = cost computed from the same list price -> deviation ~1.0 by # construction, any drift is reprice noise. Only flag drift where the # signal is real, else the badge is non-actionable and trains the operator # to ignore it. signal = "reported" if 2 * a["n_reported"] >= a["calls"] else "derived" rows.append({ "provider": a["provider"], "calls": a["calls"], "signal": signal, "measured_usd_per_mtok": round(a["measured_usd"] / mtok, 4), "expected_usd_per_mtok": round(a["expected_usd"] / mtok, 4), "measured_usd": round(a["measured_usd"], 4), "expected_usd": round(a["expected_usd"], 4), "deviation": round(deviation, 3), "warn": signal == "reported" and a["calls"] >= min_calls and abs(deviation - 1) >= threshold, }) rows.sort(key=lambda x: abs(x["deviation"] - 1), reverse=True) return rows @app.get("/dashboard/api/cost-accuracy") async def dashboard_cost_accuracy(request: Request) -> Response: """Measured effective cost (from the calls ledger) vs the advertised list price per provider, with a drift flag. Read-only observability — never touches routing. The signal is strongest for providers that report their own cost (e.g. openrouter); a compute-from-price provider reads ~1.0 by construction. Admin-only.""" caller, error = _require_admin_dashboard_caller(request) if error: return error try: window_s = int(request.query_params.get("window_s") or 86_400) except ValueError: window_s = 86_400 window_s = max(3_600, min(window_s, 90 * 86_400)) rt = await _fetch_router_runtime() or {} ema = rt.get("ema_metrics") or {} def mult_of(provider: str) -> float: key = f"{provider}.price_multiplier" return settings.get(key) if key in settings.SCHEMA else 1.0 rows = _cost_accuracy_rows(host_store.cost_by_route(window_s), ema, mult_of) return JSONResponse(content={"rows": rows, "window_s": window_s, "generated_at": int(time.time())}) @app.get("/dashboard/api/config") async def dashboard_get_config(request: Request) -> Response: """Operator-tunable knobs per provider (antseed top-N, codex scarcity ramp, runway thresholds) with current value + default + range. Admin-only.""" caller, error = _require_admin_dashboard_caller(request) if error: return error settings.reload() return JSONResponse(content={"knobs": settings.current()}) @app.post("/dashboard/api/config") async def dashboard_set_config(request: Request) -> Response: """Set/clear knob overrides (validated against the schema), persist to the PVC and hot-apply on the router. `{updates: {key: value|null}}`; null clears an override back to its default. Admin-only.""" caller, error = _require_admin_dashboard_caller(request) if error: return error try: body = await request.json() except Exception: return JSONResponse(status_code=400, content={"error": {"message": "invalid JSON body", "type": "invalid_request", "code": "config"}}) updates = body.get("updates") if isinstance(body, dict) else None if not isinstance(updates, dict): return JSONResponse(status_code=400, content={"error": {"message": "expected {updates: {key: value}}", "type": "invalid_request", "code": "config"}}) applied, errors = settings.validate_and_write(updates) if errors: return JSONResponse(status_code=400, content={"error": {"message": "; ".join(errors), "type": "invalid_request", "code": "config_invalid"}}) applied_live = False note = None try: if _client is not None: r = await _client.post(f"{UPSTREAM}/x/config/reload", timeout=10.0, headers=_router_admin_headers()) applied_live = r.status_code == 200 if not applied_live: note = f"router /x/config/reload returned {r.status_code}" except Exception as exc: note = str(exc) _log({"event": "dashboard_config_updated", "viewer": caller, "keys": sorted(updates), "applied_live": applied_live}) return JSONResponse(content={"ok": True, "knobs": settings.current(), "applied_live": applied_live, "note": note}) @app.get("/dashboard/api/keys/reveal") async def dashboard_reveal_keys(request: Request) -> Response: caller, error = _require_admin_dashboard_caller(request) if error: return error try: consumer = _safe_consumer_name(request.query_params.get("consumer", "")) except ValueError as exc: return JSONResponse(status_code=400, content={"error": {"message": str(exc), "type": "invalid_request_error", "code": "invalid_consumer"}}) plaintext_rows = _plaintext_key_rows_for_consumer(consumer) hash_only_count = sum(1 for owner in CALLER_KEY_HASHES.values() if owner == consumer) _log({"event": "dashboard_key_revealed", "consumer": consumer, "viewer": caller, "plaintext_key_count": len(plaintext_rows), "hash_only_count": hash_only_count}) return JSONResponse(content={ "consumer": consumer, "keys": plaintext_rows, "hash_only_count": hash_only_count, "message": None if plaintext_rows else "No recoverable raw key is stored for this application. Hash-only keys cannot be revealed; generate a replacement key and copy it once.", "warning": "Admin-only endpoint. Raw keys are returned only for legacy plaintext CALLER_KEYS_JSON entries; provider credentials and hash-only consumer keys are not exposed.", }) @app.get("/dashboard/api/keys/list") async def dashboard_list_keys(request: Request) -> Response: """Per-key metadata for one consumer — drives the drawer's keys list. Admin-only, consumer-scoped, and never returns a raw key: only the hashed metadata already persisted (prefix, status, timestamps) plus a `recoverable` flag so the UI can offer reveal only where a plaintext key actually exists.""" caller, error = _require_admin_dashboard_caller(request) if error: return error try: consumer = _safe_consumer_name(request.query_params.get("consumer", "")) except ValueError as exc: return JSONResponse(status_code=400, content={"error": {"message": str(exc), "type": "invalid_request_error", "code": "invalid_consumer"}}) records = _issued_consumer_records() meta = records.get(consumer, _normalize_consumer_record(consumer, {})) recoverable_prefixes = { hashlib.sha256(token.encode()).hexdigest()[:12] for token, owner in CALLER_KEYS.items() if owner == consumer } keys = [] for row in _active_key_rows(meta.get("keys") or []): prefix = str(row.get("sha256_prefix") or "") keys.append({ "sha256_prefix": prefix, "status": row.get("status") or "active", "created_at": row.get("created_at"), "expires_at": row.get("expires_at"), "replaced_at": row.get("replaced_at"), "revoked_at": row.get("revoked_at"), "recoverable": prefix in recoverable_prefixes, }) return JSONResponse(content={ "consumer": consumer, "status": meta.get("status") or "inactive", "allowed_routes": meta.get("allowed_routes") or [], "rate_per_min": meta.get("rate_per_min"), "burst": meta.get("burst"), "batch": meta.get("batch"), "member_id": meta.get("member_id"), "budget_usd": meta.get("budget_usd"), "keys": keys, }) @app.post("/dashboard/api/keys") async def dashboard_create_key(request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error caller = str(ctx.get("viewer")) try: data = await request.json() consumer = _safe_consumer_name(str(data.get("consumer", ""))) if isinstance(data, dict) else "" rotate = bool(data.get("rotate")) if isinstance(data, dict) else False grace_period_s = _optional_int(data.get("grace_period_s"), min_value=0, max_value=90 * 24 * 3600) if isinstance(data, dict) else None rotate_prefix = str(data.get("sha256_prefix") or "").strip().lower() if isinstance(data, dict) else "" if rotate_prefix and not re.fullmatch(r"[a-f0-9]{8,64}", rotate_prefix): raise ValueError("sha256_prefix must be 8-64 lowercase hex chars") except ValueError as exc: return JSONResponse(status_code=400, content={"error": {"message": str(exc), "type": "invalid_request_error", "code": "invalid_consumer"}}) except Exception: return JSONResponse(status_code=400, content={"error": {"message": "invalid JSON body", "type": "invalid_request_error", "code": "invalid_json"}}) now = int(time.time()) token = f"{DASHBOARD_KEY_PREFIX}_{secrets.token_urlsafe(32)}" token_hash = hashlib.sha256(token.encode()).hexdigest() records = _writable_consumer_records() if records is None: return _consumer_store_unavailable() meta = records.get(consumer, _normalize_consumer_record(consumer, {})) if rotate: # With sha256_prefix, rotation targets ONLY that active key; without it, # the legacy behavior stands — every active key without an expiry is # marked to expire after the grace period. expires_at = now + (DEFAULT_ROTATION_GRACE_S if grace_period_s is None else grace_period_s) rotated_any = False for key in meta.get("keys") or []: key_prefix = str(key.get("sha256_prefix") or "") if key_prefix == token_hash[:12]: continue if rotate_prefix and not (key_prefix.startswith(rotate_prefix) or rotate_prefix.startswith(key_prefix)): continue if key.get("status") == "active" and not key.get("expires_at"): key["expires_at"] = expires_at key["replaced_at"] = now rotated_any = True if rotate_prefix and not rotated_any: return JSONResponse(status_code=404, content={"error": {"message": "sha256_prefix does not match an active key for this consumer", "type": "not_found", "code": "key_not_found"}}) new_hashes = dict(CALLER_KEY_HASHES) new_hashes[token_hash] = consumer meta.setdefault("keys", []) meta["keys"].append({"sha256_prefix": token_hash[:12], "status": "active", "created_at": now, "viewer": caller}) meta["status"] = meta.get("status") or "active" meta["updated_at"] = now records[consumer] = meta if not _write_issued_consumer_records( records, key_digests={token_hash: consumer}): return JSONResponse(status_code=500, content={"ok": False, "error": "failed to persist the issued key"}) _publish_local_consumer_map( "CALLER_KEYS_SHA256_JSON", new_hashes, CALLER_KEY_HASHES) _log({"event": "dashboard_key_created", "consumer": consumer, "viewer": caller, "rotate": rotate, "rotated_prefix": rotate_prefix or None}) return JSONResponse(content={"ok": True, "consumer": consumer, "api_key": token, "sha256_prefix": token_hash[:12], "rotate": rotate, "grace_period_s": None if not rotate else (DEFAULT_ROTATION_GRACE_S if grace_period_s is None else grace_period_s), "warning": "Copy now. The raw key is shown once and only hashed key metadata is persisted."}) @app.post("/dashboard/api/keys/batch") async def dashboard_create_key_batch(request: Request) -> Response: ctx, error = _require_admin_dashboard_auth(request) if error: return error try: data = await request.json() if not isinstance(data, dict): raise ValueError("JSON body must be an object") batch = _safe_consumer_name(str(data.get("batch", ""))) try: count = int(data.get("count")) except (TypeError, ValueError): count = 0 budget_usd = _optional_float(data.get("budget_usd"), min_value=0.01) rate_per_min = _optional_int(data.get("rate_per_min"), min_value=1) burst = _optional_int(data.get("burst"), min_value=1) allowed_routes = _clean_route_list(data.get("allowed_routes")) if not 1 <= count <= 50: raise ValueError("count must be between 1 and 50") if budget_usd is None: raise ValueError("budget_usd must be a positive number") members = [(f"{idx:03d}", _safe_consumer_name(f"{batch}-{idx:03d}")) for idx in range(1, count + 1)] except ValueError as exc: return JSONResponse(status_code=400, content={"error": { "message": str(exc), "type": "invalid_request_error", "code": "invalid_key_batch"}}) except Exception: return JSONResponse(status_code=400, content={"error": { "message": "invalid JSON body", "type": "invalid_request_error", "code": "invalid_json"}}) records = _writable_consumer_records() if records is None: return _consumer_store_unavailable() collisions = [consumer for _, consumer in members if consumer in records] if collisions: return JSONResponse(status_code=409, content={"error": { "message": "batch consumers already exist", "type": "conflict", "code": "consumer_batch_exists", "consumers": collisions[:10]}}) now = int(time.time()) viewer = str(ctx.get("viewer")) old_hashes = dict(CALLER_KEY_HASHES) new_hashes = dict(old_hashes) issued = [] for member_id, consumer in members: token = f"{DASHBOARD_KEY_PREFIX}_{secrets.token_urlsafe(32)}" digest = hashlib.sha256(token.encode()).hexdigest() new_hashes[digest] = consumer meta = _normalize_consumer_record(consumer, { "status": "active", "batch": batch, "member_id": member_id, "budget_usd": budget_usd, "rate_per_min": rate_per_min, "burst": burst, "allowed_routes": allowed_routes, "keys": [{"sha256_prefix": digest[:12], "status": "active", "created_at": now, "viewer": viewer}], "updated_at": now}) records[consumer] = meta issued.append({"batch": batch, "member_id": member_id, "consumer": consumer, "api_key": token, "sha256_prefix": digest[:12], "budget_usd": budget_usd}) created_hashes = { digest: owner for digest, owner in new_hashes.items() if digest not in old_hashes } if not _write_issued_consumer_records( records, key_digests=created_hashes): return JSONResponse(status_code=500, content={"error": { "message": "failed to persist key batch", "type": "server_error", "code": "key_batch_persistence"}}) _publish_local_consumer_map( "CALLER_KEYS_SHA256_JSON", new_hashes, CALLER_KEY_HASHES) _log({"event": "dashboard_key_batch_created", "batch": batch, "count": count, "budget_usd": budget_usd, "viewer": viewer}) return JSONResponse(content={"ok": True, "batch": batch, "count": count, "keys": issued, "warning": "Copy or download now. Raw keys are shown only once."}) def _ollama_show_response(model: str | None) -> dict[str, Any]: name = str(model or "profile:default").strip() or "profile:default" return { "license": "router compatibility metadata", "modelfile": f"FROM {name}\nPARAMETER num_ctx {ROUTER_CONTEXT_LENGTH}\n", "parameters": f"num_ctx {ROUTER_CONTEXT_LENGTH}", "template": "", "details": {"family": "llm-router", "families": ["llm-router"], "parameter_size": "router", "quantization_level": "router"}, "model_info": { "router.context_length": ROUTER_CONTEXT_LENGTH, "llm-router.context_length": ROUTER_CONTEXT_LENGTH, "general.architecture": "llm-router", }, } def _record_probe(**event: Any) -> None: now = int(time.time()) with _stats_lock: _stats["recent"].appendleft({"ts": now, "event": "probe", **event}) def _record_synthetic_probe_result(**event: Any) -> None: route = str(event.get("route") or event.get("requested_model") or "").strip() if not route: return ts = int(event.get("ts") or time.time()) status = int(event.get("status") or 0) row = { "ts": ts, "event": "synthetic_probe", "route": route, "requested_model": route, "status": status, "latency_ms": float(event.get("latency_ms") or 0), "provider": event.get("provider"), "model_family": event.get("model_family"), "served_model_id": event.get("served_model_id"), "decision_trace": event.get("decision_trace") if isinstance(event.get("decision_trace"), dict) else None, } with _stats_lock: _stats["synthetic_route_health"][route] = row def _extract_router_metadata(parsed: Any) -> tuple[str | None, str | None, str | None, dict[str, Any] | None]: if not isinstance(parsed, dict): return None, None, None, None xr = parsed.get("x_router") if not isinstance(xr, dict): return None, None, None, None trace = xr.get("decision_trace") if isinstance(xr.get("decision_trace"), dict) else None return xr.get("provider"), xr.get("model_family"), xr.get("served_model_id"), trace async def _synthetic_probe_once(route: str) -> None: assert _client is not None started = time.perf_counter() status = 502 provider = model_family = served_model_id = None decision_trace = None try: # Keep the probe payload as close as possible to normal Hermes traffic, # while bounding cost/latency. OpenAI-compatible providers require at # least 16 output tokens on newer GPT routes, so do not probe with tiny # max_tokens values that create false red route-health entries. payload = { "model": route, "messages": [{"role": "user", "content": "Reply exactly: pong"}], "max_tokens": 16, } headers = {"x-llm-router-caller": SYNTHETIC_PROBE_CALLER} r = await _client.post(f"{UPSTREAM}/v1/chat/completions", json=payload, headers=headers, timeout=SYNTHETIC_PROBE_TIMEOUT_S) status = r.status_code try: provider, model_family, served_model_id, decision_trace = _extract_router_metadata(r.json()) except Exception: pass except Exception as exc: decision_trace = {"attempts": [{"error_kind": "probe_error", "message": str(exc)[:160]}]} finally: latency_ms = round((time.perf_counter() - started) * 1000, 1) _record_synthetic_probe_result(route=route, status=status, latency_ms=latency_ms, provider=provider, model_family=model_family, served_model_id=served_model_id, decision_trace=decision_trace) _log({"event": "synthetic_probe", "route": route, "status": status, "latency_ms": latency_ms, "provider": provider, "model_family": model_family}) async def _synthetic_probe_loop() -> None: if SYNTHETIC_PROBE_INITIAL_DELAY_S > 0: await asyncio.sleep(SYNTHETIC_PROBE_INITIAL_DELAY_S) while True: for route in ROUTE_HEALTH_ROUTES: await _synthetic_probe_once(route) await asyncio.sleep(max(SYNTHETIC_PROBE_INTERVAL_S, 1.0)) _PROXY_SEGMENT_RE = re.compile(r"[A-Za-z0-9_.:@+~-]+") _PROXY_PROFILE_RE = re.compile(r"[A-Za-z0-9_-]{1,64}") def _proxy_path(request: Request) -> str | None: """The consumer API path to forward (`v1/...`, `/v1/...`, `api/...`), or None. An allowlist over the DECODED path, with dot/empty segments and encoded separators refused outright: the router would otherwise resolve `/v1/..%2fx/calls` (or `%2e%2e`, `./x`, `//x`) to its internal /x/* API.""" raw = (request.scope.get("raw_path") or b"").decode("latin-1").split("?", 1)[0].lower() if "%2f" in raw or "%5c" in raw or "\\" in raw: return None segs = str(request.scope.get("path") or "").split("/")[1:] if not segs or any(s in (".", "..") or not _PROXY_SEGMENT_RE.fullmatch(s) for s in segs): return None if (segs[0] in ("v1", "api") and len(segs) > 1) or ( len(segs) > 2 and segs[1] == "v1" and segs[0] != "x" and _PROXY_PROFILE_RE.fullmatch(segs[0])): return "/".join(segs) return None @app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"]) async def proxy(path: str, request: Request) -> Response: path = _proxy_path(request) if path is None: # Router-internal diagnostics (/x/*) and anything off the consumer API # surface: never proxied to callers. return JSONResponse(status_code=404, content={"error": {"message": "not found", "type": "invalid_request_error", "code": None}}) started = time.perf_counter() token = _extract_token(request) auth = await _caller_auth_async(token) caller = auth.get("caller") if not auth.get("ok"): code = auth.get("error_code") or "caller_auth" if code == "caller_auth_unavailable": status_code = 503 _metric_store_error("consumer_auth") else: status_code = 403 if code in {"caller_inactive", "caller_key_revoked", "caller_key_expired"} else 401 _record_reject(reason=code, path="/" + path, caller=caller, status=status_code, remote=request.client.host if request.client else None) _log({"event": "reject", "reason": code, "path": "/" + path, "caller": caller, "remote": request.client.host if request.client else None}) messages = { "caller_auth": "unauthorized caller", "caller_inactive": "caller is inactive", "caller_key_revoked": "caller key is revoked", "caller_key_expired": "caller key is expired", "caller_auth_unavailable": "caller authentication is temporarily unavailable", } error_type = "server_error" if status_code == 503 else "auth_error" return JSONResponse(status_code=status_code, content={"error": {"message": messages.get(code, "caller not authorized"), "type": error_type, "code": code}}) caller = str(caller) consumer_meta = auth.get("meta") or {} body = await request.body() requested_route = _requested_route_from(path, body) published_route = None if auth.get("tenant_id") is not None: # SaaS keys address published routes only. Raw policies, profiles and # pins cannot replace the workspace's published restrictions. import re try: payload = json.loads(body) if not isinstance(payload, dict): raise ValueError() except ValueError: return JSONResponse(status_code=400, content={"error": { "message": "Provide a JSON request object.", "type": "invalid_request_error"}}) raw_model = payload.get("model") or payload.get("name") use_key_default = raw_model in (None, "", "auto") if (request.method != "POST" or path not in {"v1/chat/completions", "v1/responses"} or not (use_key_default or re.fullmatch(r"route:[a-z0-9][a-z0-9-]{0,79}", requested_route or ""))): return JSONResponse(status_code=400, content={"error": { "message": "Use a published route, for example model='route:production'.", "type": "invalid_request_error", "code": "route_required"}}) from route_contract import apply_contract, PreferenceNotAllowed try: scope = ({"project_id": auth["project_id"], "environment_id": auth["environment_id"], "key_digest": auth["digest"]} if auth.get("scope_version") == 2 else {}) published_route = await control_plane_client.resolve_route( auth["tenant_id"], "__key_default__" if use_key_default else requested_route[6:], **scope) if use_key_default: requested_route = published_route.get("route") if not isinstance(requested_route, str) or not re.fullmatch(r"route:[a-z0-9][a-z0-9-]{0,79}", requested_route): raise control_plane_client.RouteUnavailable("The default route is unavailable.") payload["model"] = requested_route payload, published_route = apply_contract(payload, published_route) body = json.dumps(payload, separators=(",", ":")).encode() except (PreferenceNotAllowed, ValueError) as exc: return JSONResponse(status_code=400, content={'error': { 'message': str(exc), 'type': 'invalid_request_error', 'code': 'preference_not_allowed'}}) except control_plane_client.RouteUnavailable as exc: return JSONResponse(status_code=503, content={"error": { "message": str(exc), "type": "server_error", "code": "route_unavailable"}}) if not _route_allowed(caller, requested_route, consumer_meta): _record_reject(reason="route_not_allowed", path="/" + path, caller=caller, status=403, route=requested_route) _log({"event": "reject", "reason": "route_not_allowed", "caller": caller, "path": "/" + path, "route": requested_route}) return JSONResponse(status_code=403, content={"error": {"message": "caller is not allowed to use this route", "type": "auth_error", "code": "caller_route_not_allowed"}}) rate_result = await asyncio.to_thread(_rate_ok, caller, consumer_meta) # Compatibility for embedders/tests that supplied the pre-HA boolean hook. if isinstance(rate_result, tuple): rate_allowed, rate_store_ok, retry_after_s = rate_result else: rate_allowed, rate_store_ok, retry_after_s = bool(rate_result), True, 0.0 if not rate_store_ok: _metric_store_error("rate_limit") _record_reject(reason="rate_limit_unavailable", path="/" + path, caller=caller, status=503) return JSONResponse(status_code=503, content={"error": { "message": "caller rate-limit state is temporarily unavailable", "type": "server_error", "code": "caller_rate_limit_unavailable"}}) if not rate_allowed: _record_reject(reason="rate_limit", path="/" + path, caller=caller, status=429) _log({"event": "reject", "reason": "rate_limit", "caller": caller, "path": "/" + path}) return JSONResponse( status_code=429, headers={"Retry-After": str(max(1, math.ceil(retry_after_s)))}, content={"error": {"message": "caller rate limit exceeded", "type": "rate_limit_error", "code": "caller_rate_limit"}}) # Control-plane key limits ride the (uncached) route resolution. A subject # (`g:` / `k:`) may span keys, so its bucket and budget are # shared by every key in it, inside this caller's namespace. key_identity = (published_route or {}).get("key") key_limits = ((published_route or {}).get("limits") or {}) if key_identity else {} if key_limits.get("rate_per_min"): subject_rate = int(key_limits["rate_per_min"]) rate_allowed, rate_store_ok, retry_after_s = await asyncio.to_thread( host_store.consume_rate_token, f"{caller}|{key_identity['subject']}", subject_rate, int(key_limits.get("burst") or subject_rate)) if not rate_store_ok: _metric_store_error("rate_limit") _record_reject(reason="key_rate_limit_unavailable", path="/" + path, caller=caller, status=503) return JSONResponse(status_code=503, content={"error": { "message": "key rate-limit state is temporarily unavailable", "type": "server_error", "code": "key_rate_limit_unavailable"}}) if not rate_allowed: _record_reject(reason="key_rate_limit", path="/" + path, caller=caller, status=429) _log({"event": "reject", "reason": "key_rate_limit", "caller": caller, "path": "/" + path}) return JSONResponse( status_code=429, headers={"Retry-After": str(max(1, math.ceil(retry_after_s)))}, content={"error": {"message": "key rate limit exceeded", "type": "rate_limit_error", "code": "key_rate_limit"}}) if path == "api/show" and request.method.upper() == "POST": requested_model = None if body: try: parsed_body = json.loads(body.decode("utf-8")) if isinstance(parsed_body, dict): requested_model = parsed_body.get("name") or parsed_body.get("model") except Exception: pass latency_ms = round((time.perf_counter() - started) * 1000, 1) _record_probe(caller=caller, method=request.method, path="/" + path, status=200, latency_ms=latency_ms, requested_model=requested_model, route="metadata_probe", key_sha256_prefix=str(auth.get("digest") or "")[:12] or None) _log({"event": "metadata_probe", "caller": caller, "method": request.method, "path": "/" + path, "status": 200, "latency_ms": latency_ms, "requested_model": requested_model}) return JSONResponse(content=_ollama_show_response(requested_model)) budget_usd = _optional_float(consumer_meta.get("budget_usd"), min_value=0.01) if budget_usd is not None: spent_usd, spend_ok = await asyncio.to_thread(host_store.consumer_spend_usd, caller) if not spend_ok: return JSONResponse(status_code=503, content={"error": { "message": "consumer budget state is temporarily unavailable", "type": "server_error", "code": "consumer_budget_unavailable"}}) if spent_usd >= budget_usd: _record_reject(reason="budget_exhausted", path="/" + path, caller=caller, status=402, route=requested_route) return JSONResponse(status_code=402, content={"error": { "message": "consumer budget exhausted", "type": "budget_error", "code": "consumer_budget_exhausted", "budget_usd": budget_usd, "spent_usd": spent_usd}}) if auth.get("tenant_id") is not None and not control_plane_client.trusted_transport_ok(UPSTREAM): return JSONResponse(status_code=503, content={"error": { "message": "Trusted router bridge requires HTTPS", "type": "server_error", "code": "bridge_transport_unavailable"}}) assert _client is not None # Budget reservation is the LAST admission step before the capacity permit; # from here on every exit path must settle it (see _settle_reservation). reservation: dict[str, Any] | None = None budget_limit = key_limits.get("monthly_budget_usd") if budget_limit is not None: reservation = {"id": secrets.token_hex(16), "done": False, # A budget below one reservation still admits a first call. "amount": min(CLOUD_BUDGET_RESERVATION_USD, float(budget_limit))} allowed, store_ok, spent_usd, _reserved = await asyncio.to_thread( host_store.reserve_subject_budget, int(auth["tenant_id"]), key_identity["subject"], time.strftime("%Y-%m", time.gmtime()), float(budget_limit), reservation["amount"], reservation["id"], KEY_BUDGET_RESERVATION_TTL_S) if not store_ok: _metric_store_error("key_budget") _record_reject(reason="key_budget_unavailable", path="/" + path, caller=caller, status=503, route=requested_route) return JSONResponse(status_code=503, content={"error": { "message": "key budget state is temporarily unavailable", "type": "server_error", "code": "key_budget_unavailable"}}) if not allowed: _record_reject(reason="key_budget_exhausted", path="/" + path, caller=caller, status=402, route=requested_route) return JSONResponse(status_code=402, content={"error": { "message": "key budget exhausted", "type": "budget_error", "code": "key_budget_exhausted", "budget_usd": float(budget_limit), "spent_usd": round(spent_usd, 6)}}) def _settle_reservation(cost: float | None) -> None: # Idempotent per request; the store is idempotent per reservation too. # A failed settle leaves the reservation to expire (TTL), never to leak. if reservation is None or reservation["done"]: return reservation["done"] = True _submit_settle(reservation["id"], cost, caller) try: acquired = await _capacity_acquire() except BaseException: _settle_reservation(None) raise if not acquired: _settle_reservation(None) _record_reject(reason="router_overloaded", path="/" + path, caller=caller, status=503, route=requested_route) return JSONResponse( status_code=503, headers={"Retry-After": str(max(1, math.ceil(CAPACITY_QUEUE_TIMEOUT_S)))}, content={"error": { "message": "router capacity is temporarily exhausted; retry shortly", "type": "server_error", "code": "router_overloaded"}}) status = 502 provider = None model_family = None served_model_id = None served_by = None requested_model = None tokens_in = tokens_out = tokens_total = 0 tokens_cached = None cost_usd = None cost_basis = None decision_trace = None error_type = error_code = error_message = None record_in_finally = True capacity_released = False # The per-session meter DERIVES from this ledger (host_store.calls.session): # without the sid recorded here, /v1/session/{sid} answers 404 for everyone # even though the shim pinned the session fine. Same precedence as the shim: # an explicit body `session` wins (szc sends it there); the # X-Unhardcoded-Session header covers header-only clients (opencode plugin). session_id = request.headers.get("x-unhardcoded-session") def _finish(): nonlocal capacity_released, decision_trace if capacity_released: return capacity_released = True latency_ms = round((time.perf_counter() - started) * 1000, 1) try: if published_route: decision_trace = {**(decision_trace or {}), 'route': requested_route, 'route_revision': published_route['revision'], 'policy_id': (decision_trace or {}).get('policy_id') or published_route['policy_id'], 'routing_preference': published_route.get('routing_preference', 'default')} if auth.get('scope_version') == 2: decision_trace.update(project_id=auth['project_id'], environment_id=auth['environment_id']) if key_identity: decision_trace.update(key_id=key_identity['id'], subject=key_identity['subject']) _record_request(caller=caller, method=request.method, path="/" + path, status=status, latency_ms=latency_ms, provider=provider, model_family=model_family, served_model_id=served_model_id, served_by=served_by, requested_model=requested_model, session=session_id, tokens_in=tokens_in, tokens_out=tokens_out, tokens_total=tokens_total, tokens_cached=tokens_cached, cost_usd=cost_usd, cost_basis=cost_basis, decision_trace=decision_trace, error_type=error_type, error_code=error_code, error_message=error_message, key_sha256=auth.get("digest")) _metric_request(status, latency_ms) _log({"event": "request", "caller": caller, "method": request.method, "path": "/" + path, "status": status, "latency_ms": latency_ms, "provider": provider, "model_family": model_family}) finally: try: # Not the droppable ledger queue: a dedicated settle, retried. # A successful call whose cost is unknown (a stream the client # closed before its usage chunk, or an unpriced model) books the # reservation, never $0; failed calls book only what they report. _settle_reservation(cost_usd if cost_usd is not None or status >= 400 else reservation["amount"] if reservation else None) finally: _capacity_release() try: # Built inside the try so a failure here still reaches _finish, which # releases the capacity permit and the budget reservation. upstream_url = f"{UPSTREAM}/{path}" if request.url.query: upstream_url += f"?{request.url.query}" headers = { k: v for k, v in request.headers.items() if k.lower() not in {"authorization", "host", "connection", "content-length", "x-llm-router-tenant", "x-internal-secret", "x-unhardcoded-scope-version", "x-unhardcoded-project", "x-unhardcoded-environment", "x-unhardcoded-route", "x-unhardcoded-revision", "x-unhardcoded-policy-id", "x-unhardcoded-preference"} } headers["x-llm-router-caller"] = caller # Tenant identity for per-tenant provider credentials (BYO keys): set ONLY # from the authenticated resolve — the client-sent header is stripped above, # so it can never be smuggled past auth. if auth.get("tenant_id") is not None: headers["x-llm-router-tenant"] = str(auth["tenant_id"]) headers["x-internal-secret"] = control_plane_client.CONTROL_PLANE_INTERNAL_SECRET if auth.get("scope_version") == 2: headers["x-unhardcoded-scope-version"] = "2" headers["x-unhardcoded-project"] = str(auth["project_id"]) headers["x-unhardcoded-environment"] = str(auth["environment_id"]) headers["x-unhardcoded-route"] = requested_route headers["x-unhardcoded-revision"] = str(published_route["revision"]) headers["x-unhardcoded-policy-id"] = published_route["policy_id"] headers['x-unhardcoded-preference'] = published_route.get('routing_preference', 'default') if body: try: parsed_body = json.loads(body.decode("utf-8")) if isinstance(parsed_body, dict): requested_model = parsed_body.get("model") if parsed_body.get("session"): session_id = str(parsed_body.get("session")) except Exception: pass upstream_req = _client.build_request(request.method, upstream_url, content=body, headers=headers) r = await _client.send(upstream_req, stream=True) status = r.status_code content_type = r.headers.get("content-type", "application/json") if content_type.startswith("text/event-stream"): # SSE pass-through: forward chunks unbuffered while teeing a tail # buffer. The shim's final chat.completion.chunk carries usage and # x_router (incl. the executed cost_usd), so the request is # recorded AFTER the stream ends — with tokens, cost, provider and # full-stream latency — instead of in the outer finally. record_in_finally = False upstream_closed = False async def _close_upstream_stream() -> None: nonlocal upstream_closed if upstream_closed: return upstream_closed = True await r.aclose() # The upstream is read by a pump task, not by the client-facing # generator: a client that disconnects before the final event would # otherwise record the call at $0 and walk past budget_usd. After a # disconnect the pump keeps draining (bounded in time and bytes). queue: asyncio.Queue = asyncio.Queue(maxsize=64) client_gone_at: float | None = None drain_complete = pump_finished = False def _account(tail: bytes) -> None: nonlocal provider, model_family, served_model_id, served_by, \ tokens_in, tokens_out, tokens_total, tokens_cached, cost_usd, \ cost_basis, decision_trace, error_type, error_code, error_message try: meta = _parse_stream_tail(tail) xr = meta.get("x_router") or {} if xr: provider = xr.get("provider") model_family = xr.get("model_family") served_model_id = xr.get("served_model_id") served_by = xr.get("served_by") decision_trace = xr.get("decision_trace") if isinstance(xr.get("decision_trace"), dict) else None if isinstance(xr.get("cost_usd"), (int, float)): cost_usd = float(xr["cost_usd"]) cost_basis = xr.get("cost_basis") if isinstance(xr.get("tokens_cached"), int): tokens_cached = xr["tokens_cached"] usage = meta.get("usage") if isinstance(usage, dict): tokens_in = int(usage.get("prompt_tokens") or 0) tokens_out = int(usage.get("completion_tokens") or 0) tokens_total = int(usage.get("total_tokens") or (tokens_in + tokens_out)) err = meta.get("error") if isinstance(err, dict): error_type = err.get("type") error_code = err.get("code") error_message = err.get("message") except Exception: pass if client_gone_at is not None and not drain_complete and cost_usd is None and status < 400: # Cut off before the usage event: never book a served call # as free. Same floor Cloud key budgets reserve per call. cost_usd, cost_basis = CLOUD_BUDGET_RESERVATION_USD, "estimated_disconnect" def _client_left() -> None: nonlocal client_gone_at if client_gone_at is not None or pump_finished: return client_gone_at = time.monotonic() while not queue.empty(): queue.get_nowait() asyncio.get_running_loop().call_later(STREAM_DRAIN_TIMEOUT_S, pump.cancel) async def _pump() -> None: nonlocal drain_complete, pump_finished tail = bytearray() drained = 0 failure: BaseException | None = None try: async for chunk in r.aiter_raw(): tail.extend(chunk) _trim_sse_tail(tail) if client_gone_at is None: try: # A reader that stalls this long is treated as gone. await asyncio.wait_for(queue.put(chunk), STREAM_DRAIN_TIMEOUT_S) continue except asyncio.TimeoutError: _client_left() drained += len(chunk) if drained > STREAM_DRAIN_MAX_BYTES: break else: drain_complete = True except asyncio.CancelledError: pass except Exception as exc: # noqa: BLE001 failure = exc finally: await _close_upstream_stream() _account(bytes(tail)) _finish() pump_finished = True if client_gone_at is None: with contextlib.suppress(asyncio.TimeoutError): # None = end of stream await asyncio.wait_for(queue.put(failure), STREAM_DRAIN_TIMEOUT_S) pump = asyncio.create_task(_pump()) _stream_pumps.add(pump) pump.add_done_callback(_stream_pumps.discard) async def _passthrough(): try: while (item := await queue.get()) is not None: if isinstance(item, BaseException): raise item yield item finally: _client_left() async def _stream_background() -> None: # Also covers a client that disconnects before the first # iteration: the pump switches to bounded draining, and it — # not the client — releases the upstream socket and the permit. _client_left() return StreamingResponse( _passthrough(), status_code=status, media_type=content_type, background=BackgroundTask(_stream_background)) content = await r.aread() await r.aclose() if content_type.startswith("application/json"): try: parsed = json.loads(content.decode("utf-8")) xr = parsed.get("x_router") if isinstance(parsed, dict) else None if isinstance(xr, dict): provider = xr.get("provider") model_family = xr.get("model_family") served_model_id = xr.get("served_model_id") served_by = xr.get("served_by") decision_trace = xr.get("decision_trace") if isinstance(xr.get("decision_trace"), dict) else None if isinstance(xr.get("cost_usd"), (int, float)): cost_usd = float(xr["cost_usd"]) cost_basis = xr.get("cost_basis") if isinstance(xr.get("tokens_cached"), int): tokens_cached = xr["tokens_cached"] usage = parsed.get("usage") if isinstance(parsed, dict) else None if isinstance(parsed, dict): err = parsed.get("error") if isinstance(err, dict): error_type = err.get("type") error_code = err.get("code") error_message = err.get("message") if isinstance(usage, dict): tokens_in = int(usage.get("prompt_tokens", usage.get("input_tokens")) or 0) tokens_out = int(usage.get("completion_tokens", usage.get("output_tokens")) or 0) tokens_total = int(usage.get("total_tokens") or (tokens_in + tokens_out)) except Exception: pass return Response(status_code=status, content=content, media_type=content_type) except Exception as exc: error_type = "proxy_error" error_code = "upstream" error_message = str(exc) return JSONResponse(status_code=502, content={"error": {"message": f"upstream proxy error: {exc}", "type": "proxy_error", "code": "upstream"}}) finally: if record_in_finally: _finish() def _record_reject(**event: Any) -> None: # #5: a reject (auth fail / rate limit) is NOT an LLM call, so it does NOT go # into the `calls` ledger — it lives in the runtime recent feed only. (Before, # rejects were dual-written to usage-history; with that file retired, the # persistent stats are per-LLM-call and rejects are a runtime view.) stored = {"ts": int(time.time()), "event": "reject", **event} _metric_reject(str(event.get("reason") or "unknown")) with _stats_lock: _stats["total_rejects"] += 1 _stats["recent"].appendleft(stored) def _append_usage_history(row: dict[str, Any]) -> None: # The call ledger IS the durable store now (#5: the usage-history file is # retired; its reader derives from `calls`). Offloaded to the host store's # background writer so the DB write never sits on the request's latency path. host_store.insert_call_async(row) def _read_usage_history(*, since: int | None = None, caller: str | None = None, events: tuple[str, ...] = ("request",)) -> list[dict[str, Any]]: # The persistent usage rows for the dashboard timeframe stats, DERIVED from the # `calls` ledger now (#5: usage-history.jsonl retired). `calls` are request # events; `events` is kept for signature compatibility. # # Push the timeframe `since` (and optional caller) into the query so a windowed # view reads only its window via idx_calls_ts — not the whole retention table # to then discard almost all of it in Python. since=None ("all") still loads # the retained rows (bounded by ROUTER_DB_RETENTION_DAYS). return host_store.usage_rows(since_ts=since, caller=caller) _SSE_TAIL_HARD_CAP = 4 * 1024 * 1024 # 4 MiB backstop if no event boundary appears STREAM_DRAIN_TIMEOUT_S = max(1.0, float(os.getenv("STREAM_DRAIN_TIMEOUT_S", "120"))) STREAM_DRAIN_MAX_BYTES = max(1, int(os.getenv("STREAM_DRAIN_MAX_BYTES", str(32 * 1024 * 1024)))) _stream_pumps: set[asyncio.Task] = set() # strong refs: a drain outlives its request def _trim_sse_tail(tail: bytearray) -> None: """Bound the streamed-response tail buffer WITHOUT splitting an SSE event. The shim's final event carries usage + x_router, and its decision_trace (the ranked catalog) routinely pushes that single event past 64 KiB. The old blind byte cap (`del tail[:len-65536]`) sliced through that event's JSON, so `_parse_stream_tail` could not parse it and Activity recorded empty rows (provider/model_family/tokens/decision_trace all null) for every real call. Keep only the last three COMPLETE events (SSE separates events with a blank line, "\\n\\n"): bounded memory, yet the x_router event — usually the second-to-last, just before the terminal "[DONE]" — stays whole however large its trace. Trim only once at least three boundaries exist, so a stream with fewer events is never clipped. A generous byte cap backstops a stream that never emits a boundary. Mutates `tail` in place. """ idx = len(tail) found = 0 for _ in range(3): j = tail.rfind(b"\n\n", 0, idx) if j == -1: break idx = j found += 1 if found == 3: del tail[:idx + 2] if len(tail) > _SSE_TAIL_HARD_CAP: del tail[:len(tail) - _SSE_TAIL_HARD_CAP] def _parse_stream_tail(tail: bytes) -> dict[str, Any]: """Extract usage / x_router / error from the tail of a shim SSE stream. The shim's final chat.completion.chunk carries usage and x_router together; an aborted stream emits a single error event. Scans data lines newest-first and stops once both usage and x_router are found.""" out: dict[str, Any] = {} lines = tail.decode("utf-8", "replace").splitlines() for line in reversed(lines): line = line.strip() if not line.startswith("data:"): continue data = line[len("data:"):].strip() if not data or data == "[DONE]": continue try: payload = json.loads(data) except ValueError: continue if not isinstance(payload, dict): continue if "usage" not in out and isinstance(payload.get("usage"), dict): out["usage"] = payload["usage"] if "x_router" not in out and isinstance(payload.get("x_router"), dict): out["x_router"] = payload["x_router"] if "error" not in out and isinstance(payload.get("error"), dict): out["error"] = payload["error"] if "usage" in out and "x_router" in out: break return out def _record_request(**event: Any) -> None: status = int(event.get("status") or 0) latency_ms = float(event.get("latency_ms") or 0) tokens_in = int(event.get("tokens_in") or 0) tokens_out = int(event.get("tokens_out") or 0) tokens_total = int(event.get("tokens_total") or 0) is_error = status >= 400 now = int(event.get("ts") or time.time()) try: cost_usd = float(event["cost_usd"]) if event.get("cost_usd") is not None else None except (TypeError, ValueError): cost_usd = None if cost_usd is not None: cost_usd = max(0.0, cost_usd) # a call's cost is >= 0; never accumulate negative spend key_sha256 = str(event.get("key_sha256") or "").strip().lower() key_sha256_prefix = key_sha256[:12] if re.fullmatch(r"[a-f0-9]{64}", key_sha256) else None with _stats_lock: _stats["total_requests"] += 1 if is_error: _stats["total_errors"] += 1 _stats["total_tokens_in"] += tokens_in _stats["total_tokens_out"] += tokens_out _stats["total_tokens"] += tokens_total _stats["by_status"][str(status)] += 1 route_key = event.get("requested_model") or event.get("route") or "unknown" served_key = event.get("served_model_id") or "unknown" # Error/aux rows lack a served model_family — attribute to the requested # model instead of a single opaque "unknown" bucket (mirrors host_store # usage_aggregate's family_k fallback for the persistent view). family_key = event.get("model_family") or route_key for group_name, key in (("by_caller", event.get("caller")), ("by_provider", event.get("provider") or "unknown"), ("by_model_family", family_key), ("by_route", route_key), ("by_served_model", served_key)): c = _stats[group_name][str(key or "unknown")] c["requests"] += 1 if is_error: c["errors"] += 1 c["tokens_in"] += tokens_in c["tokens_out"] += tokens_out c["tokens_total"] += tokens_total c["latency_ms_total"] += latency_ms c["latency_ms_max"] = max(float(c["latency_ms_max"] or 0), latency_ms) c["last_seen"] = now if cost_usd is not None: c["cost_usd"] = round(float(c.get("cost_usd") or 0.0) + cost_usd, 6) caller = str(event.get("caller") or "unknown") for group_name, key in (("by_caller_provider", event.get("provider") or "unknown"), ("by_caller_model_family", family_key), ("by_caller_route", route_key), ("by_caller_served_model", served_key)): c = _stats[group_name][caller][str(key or "unknown")] c["requests"] += 1 if is_error: c["errors"] += 1 c["tokens_in"] += tokens_in c["tokens_out"] += tokens_out c["tokens_total"] += tokens_total c["latency_ms_total"] += latency_ms c["latency_ms_max"] = max(float(c["latency_ms_max"] or 0), latency_ms) c["last_seen"] = now if cost_usd is not None: c["cost_usd"] = round(float(c.get("cost_usd") or 0.0) + cost_usd, 6) _stats["by_caller_status"][caller][str(status)] += 1 if key_sha256_prefix: _stats["key_owner"][key_sha256] = caller c = _stats["by_key_sha256"][key_sha256] c["requests"] += 1 if is_error: c["errors"] += 1 c["tokens_in"] += tokens_in c["tokens_out"] += tokens_out c["tokens_total"] += tokens_total c["latency_ms_total"] += latency_ms c["latency_ms_max"] = max(float(c["latency_ms_max"] or 0), latency_ms) c["last_seen"] = now if cost_usd is not None: c["cost_usd"] = round(float(c.get("cost_usd") or 0.0) + cost_usd, 6) for group_name, key in (("by_key_provider", event.get("provider") or "unknown"), ("by_key_model_family", family_key), ("by_key_route", route_key), ("by_key_served_model", served_key)): kc = _stats[group_name][key_sha256][str(key or "unknown")] kc["requests"] += 1 if is_error: kc["errors"] += 1 kc["tokens_in"] += tokens_in kc["tokens_out"] += tokens_out kc["tokens_total"] += tokens_total kc["latency_ms_total"] += latency_ms kc["latency_ms_max"] = max(float(kc["latency_ms_max"] or 0), latency_ms) kc["last_seen"] = now _stats["by_key_status"][key_sha256][str(status)] += 1 recent_event = {k: v for k, v in event.items() if k != "key_sha256"} recent_event["usage_event_id"] = str(event.get("usage_event_id") or secrets.token_hex(12)) recent_event["key_sha256"] = key_sha256 if key_sha256_prefix else None if key_sha256_prefix: recent_event["key_sha256_prefix"] = key_sha256_prefix stored_event = {"ts": now, "event": "request", **recent_event} visible_event = {k: v for k, v in stored_event.items() if k != "key_sha256"} _stats["recent"].appendleft(visible_event) if key_sha256_prefix: _append_usage_history(stored_event) def _safe_dashboard_message(value: Any, *, limit: int = 180) -> str | None: if value is None: return None text = str(value).replace("\n", " ").replace("\r", " ").strip() for marker in ("Bearer ", "sk-", "OPENROUTER_API_KEY", "DASHBOARD_PASSWORD"): if marker in text: text = text.split(marker, 1)[0] + marker + "[REDACTED]" return text[:limit] def _event_error_kind(row: dict[str, Any]) -> str | None: for key in ("error_code", "error_type"): if row.get(key): return str(row.get(key)) trace = row.get("decision_trace") if isinstance(row.get("decision_trace"), dict) else None attempts = [] if isinstance(trace, dict): attempts = trace.get("attempts") or trace.get("decision_path") or [] for attempt in reversed(attempts): if isinstance(attempt, dict) and attempt.get("error_kind"): return str(attempt.get("error_kind")) msg = str(row.get("error_message") or "") for needle in ("no_candidates", "network_error", "bad_response", "bad_request", "auth_error", "rate_limit", "exhausted", "unknown"): if needle in msg: return needle return None def _health_summary(recent: list[dict[str, Any]], route_health: list[dict[str, Any]]) -> dict[str, Any]: chat_rows = [r for r in recent if r.get("event") == "request" and str(r.get("path") or "").startswith("/v1/chat/completions")] request_count = len(chat_rows) success_count = 0 error_kinds: dict[str, int] = defaultdict(int) status_counts: dict[str, int] = defaultdict(int) failing_recent: list[dict[str, Any]] = [] for row in chat_rows: status = int(row.get("status") or 0) status_counts[str(status)] += 1 kind = _event_error_kind(row) ok = 200 <= status < 300 and not kind if ok: success_count += 1 continue error_kinds[kind or "unknown"] += 1 if len(failing_recent) < 10: failing_recent.append({ "ts": row.get("ts"), "caller": row.get("caller"), "route": row.get("requested_model") or row.get("route"), "provider": row.get("provider"), "served_model_id": row.get("served_model_id"), "status": status, "error_kind": kind or "unknown", "error_message": _safe_dashboard_message(row.get("error_message")), }) error_count = request_count - success_count success_rate = round(success_count / request_count, 4) if request_count else None route_failures = sum(1 for r in route_health if r.get("state") in {"fail", "warn"}) route_count = len(route_health) route_ok = route_count > 0 and route_failures == 0 and all(r.get("state") == "ok" for r in route_health) if request_count and success_rate is not None and success_rate < 0.8: state = "down" if success_rate < 0.5 else "degraded" elif route_failures: state = "degraded" elif request_count == 0 and route_ok: state = "ok" elif request_count == 0: state = "unknown" else: state = "ok" return { "state": state, "request_count": request_count, "success_count": success_count, "error_count": error_count, "success_rate": success_rate, "status_counts": dict(sorted(status_counts.items())), "error_kinds": dict(sorted(error_kinds.items())), "route_failures": route_failures, "failing_recent": failing_recent, } def _route_health_snapshot(recent: list[dict[str, Any]], synthetic: dict[str, dict[str, Any]] | None = None) -> list[dict[str, Any]]: latest: dict[str, dict[str, Any]] = dict(synthetic or {}) for row in recent: route = row.get("requested_model") or row.get("route") if route and row.get("event") == "request": existing = latest.get(str(route)) if not existing or int(row.get("ts") or 0) >= int(existing.get("ts") or 0): latest[str(route)] = row out = [] for route in ROUTE_HEALTH_ROUTES: row = latest.get(route) if not row: out.append({"route": route, "state": "unknown", "source": None, "status": None, "provider": None, "served_model_id": None, "latency_ms": None, "last_seen": None}) continue status = int(row.get("status") or 0) trace = row.get("decision_trace") if isinstance(row.get("decision_trace"), dict) else None attempts = [] if isinstance(trace, dict): attempts = trace.get("attempts") or trace.get("decision_path") or [] empty_seen = any(a.get("error_kind") == "empty_response" for a in attempts if isinstance(a, dict)) error_kind = _event_error_kind(row) state = "ok" if 200 <= status < 300 and not (empty_seen or error_kind) else "warn" if 200 <= status < 300 else "fail" out.append({ "route": route, "state": state, "source": row.get("event"), "status": status, "provider": row.get("provider"), "model_family": row.get("model_family"), "served_model_id": row.get("served_model_id"), "latency_ms": row.get("latency_ms"), "last_seen": row.get("ts"), "error_kind": error_kind, "error_message": _safe_dashboard_message(row.get("error_message")), "note": "fallback after empty response" if empty_seen else None, }) return out def _counter_snapshot(d: dict[str, Any]) -> dict[str, Any]: out = dict(d) req = int(out.get("requests") or 0) out["latency_ms_avg"] = round(float(out.pop("latency_ms_total", 0.0)) / req, 1) if req else 0.0 out["latency_ms_max"] = round(float(out.get("latency_ms_max") or 0), 1) out["error_rate"] = round((int(out.get("errors") or 0) / req), 4) if req else 0.0 tokens = int(out.get("tokens_total") or 0) tokens_in = int(out.get("tokens_in") or 0) cached = int(out.get("tokens_cached") or 0) cost = float(out.get("cost_usd") or 0.0) out["cost_per_mtok"] = round(cost * 1_000_000 / tokens, 4) if tokens else None out["cost_per_request"] = round(cost / req, 6) if req else None out["cache_hit_rate"] = round(cached / tokens_in, 4) if tokens_in else None return out def _redact_usage_detail(value: Any) -> Any: if isinstance(value, dict): out = {} for key, item in value.items(): skey = str(key) if skey in {"key_sha256", "api_key", "authorization", "Authorization"}: out[skey] = "[REDACTED]" else: out[skey] = _redact_usage_detail(item) return out if isinstance(value, list): return [_redact_usage_detail(item) for item in value] if isinstance(value, str): text = value text = re.sub(r"Bearer\s+[^\s,;]+", "Bearer [REDACTED]", text) text = re.sub(r"[A-Fa-f0-9]{64}", "[REDACTED_SHA256]", text) text = re.sub(r"sk-[A-Za-z0-9_\-]{12,}", "sk-[REDACTED]", text) return text return value def _key_metadata_for_digest(digest: str, caller: str | None) -> dict[str, Any] | None: if not caller: return None meta = _consumer_meta(caller) for row in meta.get("keys") or []: prefix = str(row.get("sha256_prefix") or "").strip().lower() if prefix and (digest.startswith(prefix) or prefix.startswith(digest)): return {k: v for k, v in row.items() if k != "storage"} return {"sha256_prefix": digest[:12], "status": "active"} def _sanitize_key_usage_event(row: dict[str, Any]) -> dict[str, Any]: allowed = { "ts", "event", "caller", "method", "path", "status", "latency_ms", "provider", "model_family", "served_model_id", "requested_model", "tokens_in", "tokens_out", "tokens_total", "error_type", "error_code", "error_message", "decision_trace", "key_sha256_prefix", "cost_usd", } out = {k: _redact_usage_detail(v) for k, v in row.items() if k in allowed} if out.get("error_message") is not None: out["error_message"] = _safe_dashboard_message(out.get("error_message"), limit=500) out["error_kind"] = _event_error_kind(out) return out def _parse_usage_time(value: Any) -> int | None: if value is None or value == "": return None if isinstance(value, (int, float)): return int(value) text = str(value).strip() if not text: return None if re.fullmatch(r"\d+", text): return int(text) try: return int(datetime.fromisoformat(text.replace("Z", "+00:00")).timestamp()) except Exception as exc: raise ValueError("since/until must be a unix timestamp or ISO-8601 datetime") from exc def _parse_usage_window(value: Any, *, now: int | None = None) -> tuple[int | None, str | None]: if value is None or value == "": return None, None text = str(value).strip().lower() match = re.fullmatch(r"(\d+)([mhdw])", text) if not match: raise ValueError("window must look like 15m, 24h, 7d, or 4w") amount = int(match.group(1)) unit = match.group(2) seconds = amount * {"m": 60, "h": 3600, "d": 86400, "w": 604800}[unit] return int(now or time.time()) - seconds, text def _usage_query_options(source: Any) -> dict[str, Any]: get = source.get if hasattr(source, "get") else lambda key, default=None: default now = int(time.time()) since = _parse_usage_time(get("since")) window_since, window_label = _parse_usage_window(get("window"), now=now) if window_since is not None: since = window_since until = _parse_usage_time(get("until")) limit = _optional_int(get("limit"), min_value=1, max_value=500) offset = _optional_int(get("offset"), min_value=0, max_value=1_000_000) or 0 return {"since": since, "until": until, "window": window_label, "limit": limit or 100, "offset": offset} _price_table_cache: "tuple[tuple[str, float], dict[tuple[str, str], dict[str, float]]] | None" = None def _price_table() -> dict[tuple[str, str], dict[str, float]]: # Memoized on (path, mtime): this used to re-read and regex-parse the price # file on EVERY call (once per aggregated row set, several times per # dashboard load). Still consulted at write time and by the cost backfill. global _price_table_cache path = _resolved_path(DASHBOARD_POLICY_METRICS_PATH) try: stamp = (str(path), path.stat().st_mtime) except OSError: return {} cached = _price_table_cache if cached and cached[0] == stamp: return cached[1] try: text = path.read_text() except Exception: return {} prices: dict[tuple[str, str], dict[str, float]] = {} pattern = re.compile(r'\["(?P[^"]+)@(?P[^"]+)"\]\s*=\s*\{(?P[^}]+)\}') for match in pattern.finditer(text): body = match.group("body") pin = re.search(r"price_in_usd_per_mtok\s*=\s*([0-9.]+)", body) pout = re.search(r"price_out_usd_per_mtok\s*=\s*([0-9.]+)", body) if pin and pout: prices[(match.group("family"), match.group("provider"))] = {"input": float(pin.group(1)), "output": float(pout.group(1))} _price_table_cache = (stamp, prices) return prices def _cost_for_event(row: dict[str, Any], prices: dict[tuple[str, str], dict[str, float]]) -> tuple[float | None, dict[str, Any] | None]: # prefer the cost stamped at execution time (the price the ranker # actually used); fall back to the read-time estimate for older events stamped = row.get("cost_usd") if isinstance(stamped, (int, float)): # Clamp: a call's cost is >= 0. Events recorded before the cost was clamped # at the source carry negative spend (a negative chosen price); never let # them subtract from a consumer's / the analytics total. return max(0.0, float(stamped)), None # List-price inference estimates cannot fill a missing decision component. summary = row.get("routing_summary") or row.get("decision_trace") or {} if row.get("cost_basis") == "unknown_total" or (isinstance(summary, dict) and "automatic" in summary): return None, None family = str(row.get("model_family") or "") provider = str(row.get("provider") or "") price = prices.get((family, provider)) if not price: return None, None tokens_in = int(row.get("tokens_in") or 0) tokens_out = int(row.get("tokens_out") or 0) cost = round((tokens_in / 1_000_000.0) * price["input"] + (tokens_out / 1_000_000.0) * price["output"], 6) return max(0.0, cost), {"model_family": family, "provider": provider, "input_usd_per_mtok": price["input"], "output_usd_per_mtok": price["output"]} def _add_counter(counter: dict[str, Any], row: dict[str, Any], cost: float | None = None) -> None: status = int(row.get("status") or 0) tokens_in = int(row.get("tokens_in") or 0) tokens_out = int(row.get("tokens_out") or 0) tokens_total = int(row.get("tokens_total") or (tokens_in + tokens_out)) tokens_cached = int(row.get("tokens_cached") or 0) latency_ms = float(row.get("latency_ms") or 0) ts = int(row.get("ts") or 0) counter["requests"] += 1 if status >= 400: counter["errors"] += 1 counter["tokens_in"] += tokens_in counter["tokens_out"] += tokens_out counter["tokens_total"] += tokens_total counter["tokens_cached"] += tokens_cached counter["latency_ms_total"] += latency_ms counter["latency_ms_max"] = max(float(counter.get("latency_ms_max") or 0), latency_ms) counter["last_seen"] = max(int(counter.get("last_seen") or 0), ts) or None if cost is not None: counter["cost_usd"] = round(float(counter.get("cost_usd") or 0) + cost, 6) def _usage_events_for_key(digest: str, caller: str | None, since: int | None = None) -> tuple[list[dict[str, Any]], bool]: prefix = digest[:12] with _stats_lock: memory_rows = [dict(r) for r in _stats["recent"] if r.get("key_sha256_prefix") == prefix] history_rows = host_store.key_usage_rows(digest, since_ts=since, caller=caller) rows_by_id: dict[str, dict[str, Any]] = {} anonymous_rows: list[dict[str, Any]] = [] for row in history_rows + memory_rows: if caller and row.get("caller") != caller: continue rid = str(row.get("usage_event_id") or "") if rid: rows_by_id[rid] = row else: anonymous_rows.append(row) rows = list(rows_by_id.values()) + anonymous_rows rows.sort(key=lambda r: int(r.get("ts") or 0), reverse=True) return rows, bool(history_rows) def _windowed_usage_events(rows: list[dict[str, Any]], opts: dict[str, Any]) -> list[dict[str, Any]]: since = opts.get("since") until = opts.get("until") out = [] for row in rows: ts = int(row.get("ts") or 0) if since is not None and ts < int(since): continue if until is not None and ts > int(until): continue out.append(row) return out def _period_totals(rows: list[dict[str, Any]], *, monthly: bool) -> list[dict[str, Any]]: buckets: dict[str, dict[str, Any]] = {} prices = _price_table() for row in rows: ts = int(row.get("ts") or 0) dt = datetime.fromtimestamp(ts, tz=timezone.utc) key = dt.strftime("%Y-%m" if monthly else "%Y-%m-%d") b = buckets.setdefault(key, {"month" if monthly else "date": key, **_counter()}) cost, _ = _cost_for_event(row, prices) _add_counter(b, row, cost) return [{k: v for k, v in _counter_snapshot(b).items() if k != "last_seen"} | {"month" if monthly else "date": key} for key, b in sorted(buckets.items(), reverse=True)] def _key_usage_snapshot(*, viewer: str, key_sha256: str, caller: str | None = None, options: dict[str, Any] | None = None) -> dict[str, Any]: digest = str(key_sha256 or "").strip().lower() if not re.fullmatch(r"[a-f0-9]{64}", digest): raise ValueError("key_sha256 must be a 64-character lowercase hex digest") prefix = digest[:12] options = options or {"since": None, "until": None, "window": None, "limit": 100, "offset": 0} with _stats_lock: inferred_caller = caller or _stats["key_owner"].get(digest) or CALLER_KEY_HASHES.get(digest) all_rows, persistent = _usage_events_for_key(digest, inferred_caller, since=options.get("since")) rows = _windowed_usage_events(all_rows, options) prices = _price_table() counter = _counter() by_provider: dict[str, dict[str, Any]] = defaultdict(_counter) by_model_family: dict[str, dict[str, Any]] = defaultdict(_counter) by_route: dict[str, dict[str, Any]] = defaultdict(_counter) by_served_model: dict[str, dict[str, Any]] = defaultdict(_counter) by_status: dict[str, int] = defaultdict(int) total_cost = 0.0 priced_events = 0 price_sources: dict[str, dict[str, Any]] = {} enriched_rows: list[dict[str, Any]] = [] for row in rows: cost, price_meta = _cost_for_event(row, prices) if cost is not None: priced_events += 1 total_cost = round(total_cost + cost, 6) row = {**row, "cost_usd": cost} if price_meta: price_sources[f"{price_meta['model_family']}@{price_meta['provider']}"] = price_meta _add_counter(counter, row, cost) _add_counter(by_provider[str(row.get("provider") or "unknown")], row, cost) _add_counter(by_model_family[str(row.get("model_family") or "unknown")], row, cost) _add_counter(by_route[str(row.get("requested_model") or row.get("route") or "unknown")], row, cost) _add_counter(by_served_model[str(row.get("served_model_id") or "unknown")], row, cost) by_status[str(int(row.get("status") or 0))] += 1 enriched_rows.append(row) recent_total = len(enriched_rows) offset = int(options.get("offset") or 0) limit = int(options.get("limit") or 100) recent_sanitized = [_sanitize_key_usage_event(dict(r)) for r in enriched_rows[offset:offset + limit]] route_health = _route_health_snapshot(recent_sanitized, {}) health_summary = _health_summary(recent_sanitized, route_health) meta = _consumer_meta(inferred_caller) if inferred_caller else {} snap_counter = _counter_snapshot(counter) return { "schema_version": 3, "kind": "router_key_usage", "detail_level": "full", "viewer": viewer, "generated_at": int(time.time()), "consumer": inferred_caller, "key_sha256_prefix": prefix, "key": _key_metadata_for_digest(digest, inferred_caller), "source": {"persistent_history": persistent, "history_path_configured": True}, "window": { "since": options.get("since"), "until": options.get("until"), "window": options.get("window"), "limit": limit, "offset": offset, "recent_total": recent_total, "recent_returned": len(recent_sanitized), }, "consumer_settings": { "status": meta.get("status"), "allowed_routes": meta.get("allowed_routes") or [], "rate_per_min": meta.get("rate_per_min") or RATE_PER_MIN, "burst": meta.get("burst") or BURST, "effective_per_min": max(int(meta.get("rate_per_min") or RATE_PER_MIN), int(meta.get("burst") or BURST)), }, "totals": { "requests": snap_counter.get("requests", 0), "errors": snap_counter.get("errors", 0), "tokens_in": snap_counter.get("tokens_in", 0), "tokens_out": snap_counter.get("tokens_out", 0), "tokens_total": snap_counter.get("tokens_total", 0), "latency_ms_avg": snap_counter.get("latency_ms_avg", 0.0), "latency_ms_max": snap_counter.get("latency_ms_max", 0.0), "error_rate": snap_counter.get("error_rate", 0.0), "last_seen": snap_counter.get("last_seen"), }, "cost_estimate": { "estimated": priced_events > 0, "usd": round(total_cost, 6), "priced_events": priced_events, "unpriced_events": max(0, len(enriched_rows) - priced_events), "source": str(_resolved_path(DASHBOARD_POLICY_METRICS_PATH)), "price_sources": dict(sorted(price_sources.items())), }, "daily_totals": _period_totals(enriched_rows, monthly=False), "monthly_totals": _period_totals(enriched_rows, monthly=True), "by_provider": {k: _counter_snapshot(v) for k, v in sorted(by_provider.items())}, "by_model_family": {k: _counter_snapshot(v) for k, v in sorted(by_model_family.items())}, "by_route": {k: _counter_snapshot(v) for k, v in sorted(by_route.items())}, "by_served_model": {k: _counter_snapshot(v) for k, v in sorted(by_served_model.items())}, "by_status": dict(sorted(by_status.items())), "route_health": route_health, "health_summary": health_summary, "recent": recent_sanitized, "security": { "sanitized": True, "raw_api_key_exposed": False, "full_sha256_exposed": False, "provider_credentials_exposed": False, }, } DASHBOARD_TIMEFRAMES = {"recent", "runtime", "1h", "24h", "7d", "30d"} def _dashboard_timeframe(value: Any) -> str: text = str(value or "recent").strip().lower() if text in {"", "history", "all"}: return "recent" if text not in DASHBOARD_TIMEFRAMES: return "recent" return text def _dashboard_timeframe_options(timeframe: str) -> dict[str, Any]: timeframe = _dashboard_timeframe(timeframe) if timeframe in {"recent", "runtime"}: return {"since": None, "until": None, "window": None, "limit": DASHBOARD_STATS_RECENT_LIMIT, "offset": 0, "timeframe": timeframe} since, window_label = _parse_usage_window(timeframe) return {"since": since, "until": None, "window": window_label, "limit": DASHBOARD_STATS_RECENT_LIMIT, "offset": 0, "timeframe": timeframe} def _counter_from_sql(c: dict[str, Any]) -> dict[str, Any]: """A host_store.usage_aggregate raw counter -> the _counter() shape the dashboard snapshots. The persistent history path never carried latency (host_store.usage_rows doesn't select latency_ms), so latency stays 0.0 — exactly what the old Python fold over those rows reported. The cost_usd key appears only when at least one row was priced, matching _add_counter.""" out = {"requests": int(c.get("requests") or 0), "errors": int(c.get("errors") or 0), "tokens_in": int(c.get("tokens_in") or 0), "tokens_out": int(c.get("tokens_out") or 0), "tokens_total": int(c.get("tokens_total") or 0), "tokens_cached": int(c.get("tokens_cached") or 0), "latency_ms_total": 0.0, "latency_ms_max": 0.0, "last_seen": c.get("last_seen") or None} if int(c.get("priced") or 0): out["cost_usd"] = round(float(c.get("cost_usd") or 0.0), 6) return out def _agg_from_sql(sql_agg: dict[str, Any], recent_rows: list[dict[str, Any]], *, selected: str | None = None) -> dict[str, Any]: """Assemble the _aggregate_usage_rows output shape from the SQL aggregates plus a LIMIT'd recent page — same keys, same bucket semantics, without materializing the window. rejects are always 0 here: they were never persisted to `calls` (see _record_reject), so the old Python fold over history rows also always counted 0.""" t = sql_agg["totals"] totals = {"requests": int(t["requests"]), "rejects": 0, "errors": int(t["errors"]), "tokens_in": int(t["tokens_in"]), "tokens_out": int(t["tokens_out"]), "tokens_total": int(t["tokens_total"]), "tokens_cached": int(t.get("tokens_cached") or 0), "cost_usd": round(float(t["cost_usd"] or 0.0), 6)} def snap(bucket: dict[str, Any]) -> dict[str, Any]: return {k: _counter_snapshot(_counter_from_sql(v)) for k, v in sorted(bucket.items())} by_caller_snap = snap(sql_agg["by_caller"]) return { "totals": totals, "by_caller": {selected: by_caller_snap.get(selected, _counter_snapshot(_counter()))} if selected else by_caller_snap, "by_caller_all": by_caller_snap, "by_provider": snap(sql_agg["by_provider"]), "by_model_family": snap(sql_agg["by_model_family"]), "by_route": snap(sql_agg["by_route"]), "by_served_model": snap(sql_agg["by_served_model"]), "by_status": {k: int(v) for k, v in sorted(sql_agg["by_status"].items())}, "recent": [{k: v for k, v in r.items() if k != "key_sha256"} for r in recent_rows], } def _daily_totals_from_sql(by_day: dict[str, Any]) -> list[dict[str, Any]]: """The _period_totals(monthly=False) shape from the SQL day buckets (UTC calendar dates, newest first, last_seen dropped).""" out = [] for date, c in sorted(by_day.items(), reverse=True): row = {k: v for k, v in _counter_snapshot(_counter_from_sql(c)).items() if k != "last_seen"} row["date"] = date out.append(row) return out def _stats_history_bundle(*, since: int | None, selected: str | None, key_filter: str | None, provider: str | None, model: str | None, recent_only: bool = False) -> dict[str, Any]: """Everything the persistent-timeframe stats snapshot derives from the store, via SQL aggregation (was: load every retained row and fold in Python, TWICE). Filter semantics replicate the old Python row filters exactly: * key_filter -> caller == selected AND key_sha256 == key_filter (when the key's consumer is unknown the old code compared caller against None, so we match caller IS NULL); * else caller == selected when a consumer is selected; * provider/model -> raw-column equality, applied to the aggregated set only. The "table" pass (consumer key rows + filter options) matches the old table_rows: the caller+key-filtered set under a key filter, the whole window otherwise — never narrowed by provider/model.""" caller = selected caller_is_null = bool(key_filter) and selected is None if recent_only: rows = host_store.usage_rows_page( caller=caller, caller_is_null=caller_is_null, consumer_sha=key_filter, provider=provider, model_family=model, limit=DASHBOARD_STATS_RECENT_LIMIT) agg = _aggregate_usage_rows(rows, selected=selected) return { "agg": agg, "keys_by_caller": agg["by_caller_all"], "filter_options": {"providers": sorted(agg["by_provider"]), "models": sorted(agg["by_model_family"])}, "daily_totals": _period_totals(rows, monthly=False), "history_events": len(rows), "history_events_all": len(rows), } filtered = host_store.usage_aggregate( since_ts=since, caller=caller, caller_is_null=caller_is_null, consumer_sha=key_filter, provider=provider, model_family=model) plain = not provider and not model if key_filter: table = filtered if plain else host_store.usage_aggregate( since_ts=since, caller=caller, caller_is_null=caller_is_null, consumer_sha=key_filter) events_all = host_store.usage_count(since_ts=since) elif selected is None and plain: table = filtered events_all = int(table["totals"]["requests"]) else: table = host_store.usage_aggregate(since_ts=since) events_all = int(table["totals"]["requests"]) recent = host_store.usage_rows_page( since_ts=since, caller=caller, caller_is_null=caller_is_null, consumer_sha=key_filter, provider=provider, model_family=model, limit=RECENT_LIMIT) return { "agg": _agg_from_sql(filtered, recent, selected=selected), "keys_by_caller": {k: _counter_snapshot(_counter_from_sql(v)) for k, v in sorted(table["by_caller"].items())}, "filter_options": {"providers": sorted(table["by_provider"].keys()), "models": sorted(table["by_model_family"].keys())}, "daily_totals": _daily_totals_from_sql(filtered["by_day"]), "history_events": int(filtered["totals"]["requests"]), "history_events_all": events_all, } def _analytics_history_bundle(*, since: int, selected: str | None, provider: str | None, model: str | None, key_filter: str | None = None) -> dict[str, Any]: sql_agg, state, ok = host_store.analytics_aggregate( since_ts=since, caller=selected, provider=provider, model_family=model, consumer_sha=key_filter) recent = host_store.usage_rows_page(caller=selected, provider=provider, model_family=model, consumer_sha=key_filter, limit=DASHBOARD_STATS_RECENT_LIMIT) agg = _agg_from_sql(sql_agg, recent, selected=selected) return { "agg": agg, "keys_by_caller": {k: _counter_snapshot(_counter_from_sql(v)) for k, v in sorted(sql_agg["by_caller"].items())}, "filter_options": {"providers": sorted(sql_agg["by_provider"]), "models": sorted(sql_agg["by_model_family"])}, "daily_totals": _daily_totals_from_sql(sql_agg["by_day"]), "history_events": int(sql_agg["totals"]["requests"]), "history_events_all": int(sql_agg["totals"]["requests"]), "analytics": {"available": ok, **state}, } def _unlanded_recent_requests(since: int | None) -> list[dict[str, Any]]: """Runtime `recent` request rows whose async ledger write has NOT landed yet. The old code merged the whole history read with the runtime deque and deduped on usage_event_id; with the aggregation in SQL, only the not-yet- persisted complement needs folding in. Rows without an id are kept as-is (they were never deduped before either).""" with _stats_lock: live = [dict(r) for r in _stats["recent"] if r.get("event") == "request"] if since is not None: live = [r for r in live if int(r.get("ts") or 0) >= since] ids = [str(r.get("usage_event_id")) for r in live if r.get("usage_event_id")] if not ids: return live floor = min(int(r.get("ts") or 0) for r in live) - 60 landed = host_store.usage_event_ids_present(ids, since_ts=floor) return [r for r in live if not r.get("usage_event_id") or str(r.get("usage_event_id")) not in landed] def _aggregate_usage_rows(rows: list[dict[str, Any]], *, selected: str | None = None) -> dict[str, Any]: # The reference Python aggregation. The dashboard stats path now computes the # same shape in SQL (_stats_history_bundle / host_store.usage_aggregate) — # tests/test_dashboard_stats_sql.py asserts the two stay identical. Keep the # semantics here and there in lockstep. totals = {"requests": 0, "rejects": 0, "errors": 0, "tokens_in": 0, "tokens_out": 0, "tokens_total": 0, "tokens_cached": 0, "cost_usd": 0.0} prices = _price_table() by_caller: dict[str, dict[str, Any]] = defaultdict(_counter) by_provider: dict[str, dict[str, Any]] = defaultdict(_counter) by_model_family: dict[str, dict[str, Any]] = defaultdict(_counter) by_route: dict[str, dict[str, Any]] = defaultdict(_counter) by_served_model: dict[str, dict[str, Any]] = defaultdict(_counter) by_status: dict[str, int] = defaultdict(int) for row in rows: # History now carries reject events alongside requests (persisted by # _record_reject); they count as rejects, never as requests, and skip # the request-shaped aggregation below. if (row.get("event") or "request") != "request": if row.get("event") == "reject": totals["rejects"] += 1 continue status = int(row.get("status") or 0) cost, _ = _cost_for_event(row, prices) totals["requests"] += 1 if status >= 400: totals["errors"] += 1 totals["tokens_in"] += int(row.get("tokens_in") or 0) totals["tokens_out"] += int(row.get("tokens_out") or 0) totals["tokens_total"] += int(row.get("tokens_total") or (int(row.get("tokens_in") or 0) + int(row.get("tokens_out") or 0))) totals["tokens_cached"] += int(row.get("tokens_cached") or 0) if cost is not None: totals["cost_usd"] = round(totals["cost_usd"] + cost, 6) by_status[str(status)] += 1 for bucket, key in ( (by_caller, row.get("caller") or "unknown"), (by_provider, row.get("provider") or "unknown"), # error/aux rows lack a served family — fall back to the requested # model (matches host_store usage_aggregate + the live _stats fold) (by_model_family, row.get("model_family") or row.get("requested_model") or row.get("route") or "unknown"), (by_route, row.get("requested_model") or row.get("route") or "unknown"), (by_served_model, row.get("served_model_id") or "unknown"), ): _add_counter(bucket[str(key or "unknown")], row, cost) by_caller_snap = {k: _counter_snapshot(v) for k, v in sorted(by_caller.items())} return { "totals": totals, "by_caller": {selected: by_caller_snap.get(selected, _counter_snapshot(_counter()))} if selected else by_caller_snap, "by_caller_all": by_caller_snap, "by_provider": {k: _counter_snapshot(v) for k, v in sorted(by_provider.items())}, "by_model_family": {k: _counter_snapshot(v) for k, v in sorted(by_model_family.items())}, "by_route": {k: _counter_snapshot(v) for k, v in sorted(by_route.items())}, "by_served_model": {k: _counter_snapshot(v) for k, v in sorted(by_served_model.items())}, "by_status": dict(sorted(by_status.items())), "recent": [{k: v for k, v in r.items() if k != "key_sha256"} for r in rows[:RECENT_LIMIT]], } def _stats_snapshot(*, viewer: str, upstream_status: int, upstream_health: dict[str, Any], consumer: str | None = None, timeframe: str = "recent", key_sha256: str | None = None, viewer_role: str = "admin", provider: str | None = None, model: str | None = None) -> dict[str, Any]: # A consumer viewer is always scoped to itself, even if the consumer store # read failed and its name is missing from _consumers(). selected = consumer if (viewer_role != "admin" or consumer in _consumers()) else None provider = (provider or "").strip() or None model = (model or "").strip() or None key_sha256 = str(key_sha256 or "").strip().lower() key_filter = key_sha256 if re.fullmatch(r"[a-f0-9]{64}", key_sha256) else None timeframe = _dashboard_timeframe(timeframe) if timeframe != "runtime": # The window aggregation runs in SQL (GROUP BY over `calls` on # idx_calls_ts) and is TTL-cached per query tuple — the 15s dashboard # auto-refresh and concurrent viewers reuse one aggregation instead of # re-scanning the window per request. since = _dashboard_timeframe_options(timeframe).get("since") cache_key = ("stats", timeframe, selected, key_filter, provider, model) if timeframe == "recent": compute = lambda: _stats_history_bundle( since=since, selected=selected, key_filter=key_filter, provider=provider, model=model, recent_only=True) else: compute = lambda: _analytics_history_bundle( since=int(since or 0), selected=selected, provider=provider, model=model, key_filter=key_filter) bundle = _snapshot_cached_compute(cache_key, compute) agg = bundle["agg"] # Surface live in-memory events (dashboard test calls, probes, just-served # requests) that are not persisted to billing history, so Activity always # shows the latest activity regardless of timeframe. Display-only: counters # and totals stay history-based. with _stats_lock: synthetic = dict(_stats["synthetic_route_health"]) live = [_sanitize_key_usage_event(dict(r)) for r in _stats["recent"] if (not selected or r.get("caller") == selected) and (not key_filter or r.get("key_sha256_prefix") == (key_filter[:12] if key_filter else None))] seen_ids = {r.get("usage_event_id") for r in agg["recent"] if r.get("usage_event_id")} merged_recent = ([r for r in live if r.get("usage_event_id") not in seen_ids] + agg["recent"]) merged_recent = merged_recent[:DASHBOARD_STATS_RECENT_LIMIT] route_health = _route_health_snapshot(agg["recent"], synthetic) health_summary = _health_summary(agg["recent"], route_health) return { "viewer": viewer, "generated_at": int(time.time()), "uptime_s": round(time.time() - _started_wall, 1), "viewer_role": viewer_role, "selected_consumer": selected, "selected_key_sha256_prefix": key_filter[:12] if key_filter else None, "timeframe": {"selected": timeframe, "source": "recent_events" if timeframe == "recent" else "analytics_hourly", "history_path_configured": True, "history_events": bundle["history_events"], "history_events_all": bundle["history_events_all"], "analytics": bundle.get("analytics")}, "rate_limit": {"rate_per_min": RATE_PER_MIN, "burst": BURST, "effective_per_min": max(RATE_PER_MIN, BURST)}, "upstream": {"status": upstream_status, "health": upstream_health}, "consumers": [{"name": name, "configured": True} for name in _consumers()], "keys": [row for row in _consumer_key_rows(bundle["keys_by_caller"]) if row.get("consumer") == selected] if key_filter and selected else _consumer_key_rows(bundle["keys_by_caller"]), "totals": agg["totals"], "by_caller": agg["by_caller"], "by_provider": agg["by_provider"], "by_model_family": agg["by_model_family"], "by_route": agg["by_route"], "by_served_model": agg["by_served_model"], "by_status": agg["by_status"], "recent": merged_recent, "filter_options": bundle["filter_options"], "daily_totals": bundle["daily_totals"], "route_health": route_health, "health_summary": health_summary, "logins": _login_connections_snapshot(timeframe="recent" if timeframe != "recent" else timeframe, consumer=selected, viewer_role=viewer_role), "provider_keys": _provider_credentials_snapshot(timeframe="recent" if timeframe != "recent" else timeframe, viewer_role=viewer_role), } with _stats_lock: by_caller_all = {k: _counter_snapshot(v) for k, v in sorted(_stats["by_caller"].items())} if selected: if key_filter: selected_counter = _counter_snapshot(_stats["by_key_sha256"].get(key_filter, _counter())) recent = [r for r in _stats["recent"] if r.get("caller") == selected and r.get("key_sha256_prefix") == key_filter[:12]] by_provider = {k: _counter_snapshot(v) for k, v in sorted(_stats["by_key_provider"].get(key_filter, {}).items())} by_model_family = {k: _counter_snapshot(v) for k, v in sorted(_stats["by_key_model_family"].get(key_filter, {}).items())} by_route = {k: _counter_snapshot(v) for k, v in sorted(_stats["by_key_route"].get(key_filter, {}).items())} by_served_model = {k: _counter_snapshot(v) for k, v in sorted(_stats["by_key_served_model"].get(key_filter, {}).items())} by_status = dict(sorted(_stats["by_key_status"].get(key_filter, {}).items())) by_caller_all = {selected: selected_counter} else: selected_counter = by_caller_all.get(selected, _counter_snapshot(_counter())) recent = [r for r in _stats["recent"] if r.get("caller") == selected] by_provider = {k: _counter_snapshot(v) for k, v in sorted(_stats["by_caller_provider"].get(selected, {}).items())} by_model_family = {k: _counter_snapshot(v) for k, v in sorted(_stats["by_caller_model_family"].get(selected, {}).items())} by_route = {k: _counter_snapshot(v) for k, v in sorted(_stats["by_caller_route"].get(selected, {}).items())} by_served_model = {k: _counter_snapshot(v) for k, v in sorted(_stats["by_caller_served_model"].get(selected, {}).items())} by_status = dict(sorted(_stats["by_caller_status"].get(selected, {}).items())) totals = { "requests": selected_counter.get("requests", 0), "rejects": 0, "errors": selected_counter.get("errors", 0), "tokens_in": selected_counter.get("tokens_in", 0), "tokens_out": selected_counter.get("tokens_out", 0), "tokens_total": selected_counter.get("tokens_total", 0), } synthetic = dict(_stats["synthetic_route_health"]) route_health = _route_health_snapshot(recent, synthetic) health_summary = _health_summary(recent, route_health) by_caller = {selected: selected_counter} else: totals = {"requests": _stats["total_requests"], "rejects": _stats["total_rejects"], "errors": _stats["total_errors"], "tokens_in": _stats["total_tokens_in"], "tokens_out": _stats["total_tokens_out"], "tokens_total": _stats["total_tokens"]} by_provider = {k: _counter_snapshot(v) for k, v in sorted(_stats["by_provider"].items())} by_model_family = {k: _counter_snapshot(v) for k, v in sorted(_stats["by_model_family"].items())} by_route = {k: _counter_snapshot(v) for k, v in sorted(_stats["by_route"].items())} by_served_model = {k: _counter_snapshot(v) for k, v in sorted(_stats["by_served_model"].items())} by_status = dict(sorted(_stats["by_status"].items())) recent = list(_stats["recent"]) synthetic = dict(_stats["synthetic_route_health"]) route_health = _route_health_snapshot(recent, synthetic) health_summary = _health_summary(recent, route_health) by_caller = by_caller_all return { "viewer": viewer, "generated_at": int(time.time()), "uptime_s": round(time.time() - _started_wall, 1), "viewer_role": viewer_role, "selected_consumer": selected, "selected_key_sha256_prefix": key_filter[:12] if key_filter else None, "timeframe": {"selected": "runtime", "source": "memory", "history_path_configured": True}, "rate_limit": {"rate_per_min": RATE_PER_MIN, "burst": BURST, "effective_per_min": max(RATE_PER_MIN, BURST)}, "upstream": {"status": upstream_status, "health": upstream_health}, "consumers": [{"name": name, "configured": True} for name in _consumers()], "keys": [row for row in _consumer_key_rows(by_caller_all) if row.get("consumer") == selected] if key_filter and selected else _consumer_key_rows(by_caller_all), "totals": totals, "by_caller": by_caller, "by_provider": by_provider, "by_model_family": by_model_family, "by_route": by_route, "by_served_model": by_served_model, "by_status": by_status, "recent": recent, "filter_options": {"providers": sorted((_stats["by_provider"] or {}).keys()), "models": sorted((_stats["by_model_family"] or {}).keys())}, "route_health": route_health, "health_summary": health_summary, "logins": _login_connections_snapshot(timeframe="runtime", consumer=selected, viewer_role=viewer_role), "provider_keys": _provider_credentials_snapshot(timeframe="runtime", viewer_role=viewer_role), } def _public_base_url() -> str: """Public OpenAI-compatible base URL shown to users in the key-handoff blurb. Override with PUBLIC_BASE_URL; defaults to the loopback dev address.""" return (os.getenv("PUBLIC_BASE_URL") or "http://127.0.0.1:8080/v1").rstrip("/") def _dashboard_html() -> str: html = """ unhardcoded dashboard

Analytics

Spend, traffic and errors — filter by timeframe, consumer, provider and model.
Loading recent activity…
Applying the selected filters.
""" return html.replace("__PUBLIC_BASE_URL__", _public_base_url())