# AI-Monitoring configuration — copy to .env and fill in.
# .env is git-ignored. NEVER commit real keys.

# --- listen / storage --------------------------------------------------------
MONITOR_HOST=0.0.0.0
MONITOR_PORT=9925
MONITOR_DB_PATH=/data/ai-monitoring.db

# --- sampling ----------------------------------------------------------------
MONITOR_SAMPLE_INTERVAL=5
MONITOR_RETENTION_SAMPLES=8640
MONITOR_DB_RETENTION_HOURS=720
# How long full per-tick JSON snapshots are kept in the `samples` table (used to warm-load
# recent history on boot). Hours; default 12.
MONITOR_SAMPLES_RETENTION_HOURS=12
MONITOR_HTTP_TIMEOUT=4
# Max bytes read from a single collector response (defends against a compromised
# or MITM'd backend returning a huge body and OOM-ing the monitor). Default 16 MiB.
MONITOR_HTTP_MAX_BYTES=16777216
# Server-Sent Events (/api/stream) concurrency caps — a new stream is refused with 429 once
# a limit is hit, so a fleet of open dashboards can't exhaust connections. Total across all
# clients (default 200) and per-client (default 12).
MONITOR_SSE_MAX_TOTAL=200
MONITOR_SSE_MAX_PER_CLIENT=12
# Friendly name for THIS machine, shown in every alert message ("[<name>] AI-Monitoring — …").
# Empty = use the monitored host's own hostname.
MONITOR_INSTANCE_NAME=
# 1 = log each collector's availability + error to `docker logs` (why a panel is
# missing, e.g. GPU). Logs on startup + on change. Back-compat alias: forces the
# global level to DEBUG (same as MONITOR_LOG_LEVEL=debug). Leave 0 in normal use.
MONITOR_DEBUG=0

# --- structured logging (obslog) --------------------------------------------
# Global log level: debug | info | warning | error. Default info. INFO shows
# startup/backend/alert events; DEBUG adds the per-poll firehose (noisy).
MONITOR_LOG_LEVEL=info
# Output shape: text (human, default) or json (one object per line, for a log
# aggregator / Loki / ELK). Secrets are auto-redacted in both.
MONITOR_LOG_FORMAT=text
# Collapse identical repeated lines within this many seconds (0 = off). Default 0.
MONITOR_LOG_DEDUPE_S=0
# Per-component override, e.g. MONITOR_LOG_LEVEL_aimon.litellm=debug to raise ONE
# logger without flooding the rest. Components: access spend litellm sampler auth
# collector db.

# --- optional dashboard auth (Bearer token / ?token=). Empty = open. ---------
# Use ≥16 random chars (a short token is flagged at boot). Rate-limited: after
# AUTH_MAX_FAILS bad tokens from one IP within AUTH_WINDOW_S, that IP is locked
# (429) for AUTH_LOCKOUT_S. Behind a reverse proxy set AUTH_TRUSTED_PROXY=1 so the
# real client IP is read from X-Forwarded-For — the proxy must APPEND (not replace)
# it, since the collector trusts the rightmost hop (F4).
MONITOR_DASHBOARD_TOKEN=
MONITOR_AUTH_MAX_FAILS=10
MONITOR_AUTH_WINDOW_S=300
MONITOR_AUTH_LOCKOUT_S=900
# Per-ACCOUNT lockout (independent of the per-IP one above): after
# AUTH_USER_MAX_FAILS bad password attempts on the same username within
# AUTH_WINDOW_S, that account is locked for AUTH_USER_LOCKOUT_S — protects a
# targeted account even when the attacker rotates source IPs.
MONITOR_AUTH_USER_MAX_FAILS=10
MONITOR_AUTH_USER_LOCKOUT_S=300
MONITOR_AUTH_TRUSTED_PROXY=0
# F2: with no token the whole dashboard + API is open — refused at boot unless you
# opt in here (loopback / behind an authenticating proxy only).
MONITOR_ALLOW_OPEN=0
# F3: the session cookie carries the token, so it is marked Secure (HTTPS-only) by
# default. Set to 1 ONLY for local plain-HTTP testing.
MONITOR_COOKIE_ALLOW_INSECURE=0

# --- multi-user access (username + password login; SQLite-backed) -------------
# External users log in with their own credentials instead of sharing the token.
# The FIRST admin is seeded from these on an empty users table (idempotent — once
# any user exists these are ignored). After first boot, manage users at
# /admin/users. Roles: 'admin' (manage users) or 'viewer' (read-only).
MONITOR_ADMIN_USER=
MONITOR_ADMIN_PASSWORD=
MONITOR_ADMIN_EMAIL=
# How long a login session lasts before re-auth (seconds; default 7 days).
MONITOR_SESSION_TTL_S=604800
# Hard ceiling on concurrent server-side sessions (bounds in-memory session
# stores; oldest-expiring are evicted past this). Default 2000.
MONITOR_SESSION_MAX=2000
# How long the access/admin audit trail is kept (days). Admins review it in the
# Activity log section of /admin/users.
MONITOR_AUDIT_RETENTION_DAYS=90

# --- Prometheus /metrics export ---------------------------------------------
# GET /metrics exposes aimon_* gauges for Prometheus/Grafana/AlertManager and
# fleet aggregation. Gated like the API; give Prometheus a scrape-only token.
MONITOR_METRICS_ENABLED=1
MONITOR_METRICS_TOKEN=

# --- LiteLLM (JSON only, no prometheus) --------------------------------------
LITELLM_BASE_URL=http://CHANGE_ME:4000
LITELLM_MASTER_KEY=sk-CHANGE_ME
LITELLM_SPEND_WINDOW_MIN=15
# LiteLLM counts the monitor's own /health/backlog probe as in-flight → idle proxy
# reports backlog=1. 1 = subtract that self-probe (idle reads 0); 0 = don't.
MONITOR_LITELLM_BACKLOG_PROBE_SELFCOUNT=1
# Hide specific keys/users from EVERY per-key & per-user graph (Top keys, Top users,
# Cost by user, model×user, keys-over-time). Comma-separated, case-insensitive; match a
# key ALIAS, key hash, or resolved owner email/username. Prefer the key alias/hash.
#
# SET THIS if the LiteLLM key the monitor authenticates with is a normal key that also
# appears in LiteLLM's own accounting: it otherwise occupies a slot in every per-key
# ranking and its historic spend is charted as if it were user traffic. Give it the
# alias you created for the monitor, e.g.:
#   MONITOR_EXCLUDE_KEYS=svc-monitoring
# LiteLLM's internal health-check pseudo-key is dropped automatically (it is LiteLLM
# probing its own deployments, not usage) and needs no entry here.
MONITOR_EXCLUDE_KEYS=
# Hide the "Unassigned" group — every key LiteLLM reports no owner for, with no admin
# user override — from every per-key/per-user graph. Same set the Settings board groups
# under "Unassigned"; the show/hide button on that group flips this at runtime, so this
# only sets the boot default. Hidden keys still count toward the measured totals and
# fold into "Other" (hiding drops a key's own named band, it never rewrites the total).
MONITOR_HIDE_UNASSIGNED_KEYS=0
# Restrict host container NAMES to admin sessions (non-admins still see count + health).
# The container panel reads the host docker socket; on a shared-token / multi-viewer
# deployment this keeps full host topology from every dashboard user. Default off.
MONITOR_CONTAINERS_ADMIN_ONLY=0
SLO_LATENCY_MS=2000
# Heavy calls (/health probes every deployment, /spend/logs pulls the whole day)
# poll on this slower cadence instead of every sample — protects a busy proxy.
LITELLM_HEAVY_INTERVAL=60
# Busy-proxy escape hatches: set to 0 to stop the heavy call entirely.
#   SPEND=0  -> no /spend/logs pull (drops latency/cost/key panels; keeps backlog)
#   HEALTH=0 -> no /health per-deployment probing
LITELLM_SPEND_ENABLED=1
# How to gather spend/usage:
#   full = raw /spend/logs (whole-day pull; gives latency percentiles but heavy —
#          can peg a CPU core on a busy proxy)
#   lite = server-side aggregates (/global/activity, /global/spend/keys): tiny
#          payloads, ~0 CPU, no freeze. Gives requests/tokens/cost/per-model/
#          top-keys but NO latency. RECOMMENDED for a busy proxy.
LITELLM_SPEND_MODE=full
# Adaptive load-shedding: when host 1-min load PER CORE reaches this, the monitor
# auto-STOPS the heavy calls (/health probe + full /spend/logs) and resumes when
# load drops. 0 = off. ~4 = "load is 4x the cores" (saturated). Cheap calls
# (backlog, liveliness, models, lite aggregates) keep running.
LITELLM_LOAD_SHED=0
# Hard cap on /spend/logs rows parsed per poll (most-recent kept). Bounds CPU on
# a proxy that accumulates a huge day of logs.
LITELLM_SPEND_MAX_ROWS=20000
# Timeout for the heavy calls (/health, /spend/logs). The default 4s is too short
# for a busy proxy's whole-day /spend/logs query — too low and it always times
# out and the latency/cost/key panels stay blank.
LITELLM_SPEND_TIMEOUT=20
# Circuit breaker: after this many consecutive heavy-call failures, stop calling
# for the cooldown (seconds), then probe once. Prevents the monitor from
# hammering — and freezing — a struggling proxy. Auto-recovers on success.
LITELLM_CB_THRESHOLD=3
LITELLM_CB_COOLDOWN=300
# Refuse a /spend/logs response bigger than this (bytes) before deserializing it.
LITELLM_SPEND_MAX_BYTES=67108864
# Set to 1 for verbose per-call LiteLLM logging in `docker logs` (diagnostics).
LITELLM_DEBUG=0
# Model prefixes/name-tokens treated as self-hosted -> their cost is REFERENCE
# (an estimated value shown on the Spend & Quota page, never real cash, never
# consumes a budget). Everything else counts as REAL, budgetable spend. If you
# assign your own per-token prices to local models and want them counted as real
# budgetable spend, set this EMPTY (MONITOR_INTERNAL_PROVIDERS=).
MONITOR_INTERNAL_PROVIDERS=ollama,llama-cpp,llama_cpp,llamacpp,vllm,huggingface,hf,gpt-oss,local,self-hosted,text-completion-openai
# Open-weight model FAMILIES that are self-hosted here even without a provider prefix
# (e.g. a bare "gemma4" / "qwen2.5"). Matched as a substring of the model name. Set
# EMPTY to disable and rely on the provider prefixes above only.
MONITOR_INTERNAL_MODEL_FAMILIES=gemma,qwen,mistral,mixtral,deepseek,starcoder,codellama,command-r,granite,phi-,phi3,phi4,yi-,llama
# Optional per-key MONTHLY budgets as JSON {"key-alias": <amount>, ...} — drives the
# Spend & Quota panel until a real max_budget is read from LiteLLM /key/info. Use your
# own figures in .env (never commit real budget amounts); empty = read from LiteLLM only.
MONITOR_KEY_BUDGETS=

# --- per-backend monitoring toggles (also in Settings → Services) -------------
# 0 = don't poll this backend: hidden from the sidebar + Spend, no down-alert.
MONITOR_LITELLM_ENABLED=1
MONITOR_OLLAMA_ENABLED=1
MONITOR_LLAMACPP_ENABLED=1
MONITOR_VLLM_ENABLED=1

# --- Ollama ------------------------------------------------------------------
OLLAMA_BASE_URL=http://CHANGE_ME:11434

# --- llama.cpp server --------------------------------------------------------
LLAMACPP_BASE_URL=http://CHANGE_ME:8080
LLAMACPP_API_KEY=

# --- vLLM (OpenAI-compatible server) -----------------------------------------
VLLM_BASE_URL=http://CHANGE_ME:8000
VLLM_API_KEY=
# Read vLLM's OWN /metrics endpoint for the live counters (queue depth, KV-cache %,
# TTFT, throughput, preemptions). This needs no Prometheus server, exporter or agent —
# it is vLLM reporting its own state, like llama.cpp serving /slots. Set 0 to use only
# the JSON endpoints, which can show status + loaded model but none of the counters.
VLLM_METRICS_ENABLED=1

# Host network (Ethernet dashboard) — down/up speed + totals from /proc/net/dev.
# EMPTY = auto-select physical NICs (skip loopback / veth / docker / bridges /
# overlay VPNs like tailscale/wireguard). Set a comma-separated list to pin exactly
# the interface(s) that matter, e.g. NETWORK_IFACES=eth0,eth1
NETWORK_IFACES=
# /proc/net is per-network-namespace, so in a container it shows only the container's
# own eth0 (Docker bridge), NOT the host NICs. To monitor the HOST the collector reads
# PID 1's netns (/proc/1/net/dev) — works when the container shares the host PID
# namespace (pid: host, the compose default) or on bare metal; otherwise it falls back
# to this container's netns and the dashboard flags it. Override the exact path only for
# unusual setups (e.g. a bind-mounted host proc): MONITOR_NET_DEV=/host/proc/1/net/dev
MONITOR_NET_DEV=

# Container liveness + alive-time (Docker API). Requires the Docker socket mounted
# + group_add in docker-compose.yml. EMPTY = show ALL host containers (auto-
# discover). Set a comma-separated list to watch only specific ones.
MONITOR_CONTAINERS=
# e.g. MONITOR_CONTAINERS=llama-cpp-server,new_open-webui,new_litellm,new_caddy,new_ollama
# F1 (recommended): reach Docker through a read-only socket proxy over TCP instead
# of mounting the host-root socket. docker-compose.yml ships a docker-socket-proxy
# service; point this at it. When set, the raw socket is NOT mounted into the
# monitor, so a monitor compromise can't reach the full host-root Docker API.
MONITOR_DOCKER_API_URL=http://docker-socket-proxy:2375
# Legacy direct-socket fallback (used only when MONITOR_DOCKER_API_URL is empty).
MONITOR_DOCKER_SOCKET=/var/run/docker.sock
# GID of the host 'docker' group, so the non-root monitor can read the socket.
# Find it on the host with:  getent group docker | cut -d: -f3
DOCKER_GID=999

# --- GPU (pick ONE mode; leave all empty to try a local GPU CLI) -------------
# Mode 0 (SAFEST — LOCAL host GPU): the host writes nvidia-smi CSV to a file on a
#   timer (see deploy/gpu-metrics.service) and the monitor reads it, bind-mounted
#   READ-ONLY. No SSH, no network, no shell into the host. Best when the GPU is on
#   the SAME host as this container (Alpine can't run a passed-through nvidia-smi).
GPU_METRICS_FILE=
# GPU_METRICS_FILE=/gpu/gpu.csv          # mount host dir read-only: - /var/lib/aimon-gpu:/gpu:ro
# Max age (s) before a file counts as stale (panel → unavailable). The writer
# updates every ~5s, so keep this small (~30). A large value hides a dead writer:
# a stale CSV frozen on an idle 0% reading would keep showing 0% for that long.
GPU_FILE_MAX_AGE=30
# Mode 1 (agentless SSH): monitor runs nvidia-smi on the remote GPU host.
#   Mount the key into the container and point GPU_SSH_KEY at it.
GPU_SSH=
# GPU_SSH=user@gpuhost
GPU_SSH_PORT=22
GPU_SSH_KEY=
# GPU_SSH_KEY=/keys/gpu_id_ed25519
# Mode 2 (HTTP agent): a small endpoint on the GPU box returning GPU JSON.
GPU_METRICS_URL=
# GPU_METRICS_URL=http://gpuhost:9835/gpu

# --- alert thresholds (0 disables each) --------------------------------------
ALERT_CPU_PCT=0
ALERT_MEM_PCT=0
ALERT_DISK_PCT=0
ALERT_GPU_PCT=0
ALERT_VRAM_PCT=0
ALERT_LLM_WAIT_MS=0
# vLLM queue depth: `running` is busy, `waiting` is QUEUED — the saturation signal.
ALERT_VLLM_WAITING=0
ALERT_BACKLOG=0
ALERT_ON_BACKEND_DOWN=1
ALERT_REPEAT_MIN=30
# Anti-flap hysteresis for backend up/down: fail this many consecutive polls before a backend is
# called DOWN, and answer this many consecutive polls before a DOWN one is called back UP. Raising
# these tames a backend whose /health flaps under load (e.g. LiteLLM), so it stops spamming the
# webhook + the "Recent deliveries" list with down/up pairs.
ALERT_BACKEND_DOWN_AFTER=3
ALERT_BACKEND_UP_AFTER=3
# Also call a backend DOWN when MORE THAN HALF of the last N samples failed, even without
# ALERT_BACKEND_DOWN_AFTER *consecutive* failures — catches a backend flapping badly enough to
# never string together N in a row. N samples; 0 disables. Default 10.
ALERT_BACKEND_DOWN_WINDOW=10

# Maintenance windows: suppress DOWN/recovery alerts for a KNOWN, expected outage
# (e.g. a daily vLLM model-reload restart) without touching the dashboard data —
# it still shows the backend down during the window, only the alert is silenced.
# One var per backend (LITELLM/OLLAMA/LLAMACPP/VLLM/GPU), comma-separated
# HH:MM-HH:MM (UTC, 24h, daily-recurring); a window may cross midnight
# (e.g. 23:50-00:10). Empty/unset = never suppressed.
MONITOR_MAINTENANCE_VLLM=
# MONITOR_MAINTENANCE_VLLM=03:00-03:10

# --- per-key anomaly / abuse detection (0 disables) --------------------------
ANOMALY_FACTOR=4
ANOMALY_MIN_REQS=20
ANOMALY_KEY_BUDGET_HR=0

# --- alert channel (webhook) -------------------------------------------------
# Operator-set global webhook (trusted, not SSRF-checked). Fires alongside any
# per-user webhooks. Each user can also set their OWN webhook at /account.
ALERT_WEBHOOK_URL=
# Per-user webhooks are user-supplied → SSRF-guarded (private/loopback/metadata
# addresses refused). Relax/tighten:
MONITOR_WEBHOOK_ALLOW_PRIVATE=0
MONITOR_WEBHOOK_HTTPS_ONLY=0
# Optional host allowlist for user webhooks (empty = any public host):
MONITOR_WEBHOOK_ALLOW_HOSTS=
# Max per-user webhooks resolved + posted per alert tick (fan-out is also
# concurrent + time-boxed, so one slow host can't stall the sampling loop).
MONITOR_WEBHOOK_MAX_RECIPIENTS=50
# Webhook body shape. "auto" (default) picks by URL: an MS Teams Power-Automate /
# O365 URL (…logic.azure.com/…/workflows/… or …webhook.office.com…) gets the
# Adaptive-Card envelope its stock "when a webhook request is received" flow needs
# (so it renders WITHOUT any flow edits); a Slack/other URL gets the generic
# {source,text}. Force with teams | slack | generic.
MONITOR_WEBHOOK_FORMAT=auto

# --- retention rollups -------------------------------------------------------
ROLLUP_RAW_HOURS=24
ROLLUP_MIN_DAYS=30
ROLLUP_HOUR_DAYS=365

# --- cost overrides ----------------------------------------------------------
# Pin a model's real cost when LiteLLM's own price is wrong/unreliable. JSON map of
# model -> USD per 1M tokens (a blended effective rate = real spend / tokens). Highest
# precedence: wins over LiteLLM's price and the spend-anchor. Example:
# MONITOR_MODEL_COSTS={"provider/model": 0.20}
MONITOR_MODEL_COSTS=

# Currency symbol shown before money values (default "$"). Set to your billing currency,
# e.g. MONITOR_CURRENCY=€ — display label only, no FX conversion.
MONITOR_CURRENCY=$

# One-time seed for the 'cost per model & user over time' chart: days of /spend/logs
# history to backfill at first run (0 = none, chart grows forward). Default 14.
MONITOR_SPEND_MU_BACKFILL_DAYS=14

# Cost over time card reach: how many days of the cheap daily aggregates to pull. The
# 30d/12mo chart only renders its window, but the per-year + lifetime totals sum the full
# pull, so this must span the deployment's history for 'lifetime' to match per-key spend.
# Default ~5y; raise it for an older LiteLLM install.
MONITOR_SPEND_ACTIVITY_DAYS=1826

# Restrict the Spend & Quota page + /api/spend/* to admins (per-user cost attribution
# can be sensitive). Default 0 = viewers keep access.
MONITOR_SPEND_REQUIRE_ADMIN=0