PORT=11434
HOST=0.0.0.0

# Logging / telemetry
# LOG_LEVEL: DEBUG also shows the payload sent to NVIDIA. Default INFO.
LOG_LEVEL=DEBUG
# Timezone of the clock in the logs (IANA name, e.g. Europe/Rome). Default UTC.
LOG_TZ=Europe/Rome
# LOG_TZ falls back to TZ when unset, so a container-wide TZ also moves the logs.
#TZ=Europe/Rome

# --- Server tuning (Docker image) --------------------------------------------
# gunicorn workers, threads per worker, and worker timeout in seconds. The
# production compose files read these from here.
# WEB_CONCURRENCY x THREADS is the hard ceiling on requests served at once. A
# completion blocks its thread for as long as the upstream takes to generate
# (tens of seconds), so the threads cost memory but almost no CPU: size THREADS
# on the concurrency you need, not on the core count.
#WEB_CONCURRENCY=2
#THREADS=32
#GUNICORN_TIMEOUT=600
# Pooled HTTP connections kept open towards each upstream. Defaults to THREADS:
# below it, concurrent requests queue waiting for a free connection, and urllib3
# discards the surplus — each discard costs the next request a fresh TLS
# handshake with the provider.
#UPSTREAM_POOL_SIZE=32

# --- Providers ---------------------------------------------------------------
# Multi-provider (v1.3.0+): point llmproxy at a declarative providers.toml to
# serve several upstreams at once (OpenAI-compatible, Azure, Anthropic, Gemini).
# When PROVIDERS_CONFIG points to an existing file it takes precedence over the
# NVIDIA_* vars below; otherwise those vars synthesize a single NVIDIA provider
# (zero-config, backward compatible). Generate a starting file from the current
# env with:
#   make migrate-config                (local:  writes ./providers.toml)
#   make migrate-config-docker         (docker: writes ./providers.toml via the container)
#   python -m llmproxy.scripts.env_to_toml -   (prints the TOML to stdout)
# With one provider the exposed model names stay bare (unchanged); with 2+ they
# become "<provider>:<model>". A per-model alias in the TOML overrides that name.
#PROVIDERS_CONFIG=providers.toml

# Single-provider (legacy) NVIDIA configuration, used when no providers.toml.
NVIDIA_API_BASE=https://integrate.api.nvidia.com/v1
NVIDIA_API_KEY=nvapi-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

# Read timeout (seconds) toward the upstream. Non-streaming requests do not
# receive any byte until the response is complete: slow models
# (reasoning/large/queued) may exceed the default of 120. Default 120.
UPSTREAM_TIMEOUT=120

# Force streaming toward the upstream on /chat/completions even when the
# caller requests stream=false. The upstream keeps sending SSE bytes
# continuously, so the read-timeout never fires; the proxy re-aggregates the
# chunks and still returns a single JSON response. Behavior toward the
# caller is unchanged. Valid values: 1/true/yes/on. Default false.
#FORCE_UPSTREAM_STREAM=on

# Outgoing HTTP proxy toward the upstream (corporate proxy / egress). Required if
# the server reaches the Internet only through a proxy: without these variables the
# container goes direct and requests to NVIDIA hang until
# UPSTREAM_TIMEOUT fires. env_file passes them to the container, and both requests and the proxy
# use them. NO_PROXY excludes internal hosts/localhost from the proxy.
#HTTP_PROXY=http://ep.threatpulse.net:80
#HTTPS_PROXY=http://ep.threatpulse.net:80
#NO_PROXY=localhost,127.0.0.1,.hcltechlab.com,.ctg.labs,.hpelab.local,10.10.0.0/21,192.168.0.0/21

# Retry on transient upstream errors (429 / 5xx / network errors).
# RETRY_MAX: number of retries beyond the first attempt (0 = disabled). Default 2.
# RETRY_BACKOFF: base of the exponential backoff in seconds (respects Retry-After if present). Default 0.5.
RETRY_MAX=2
RETRY_BACKOFF=0.5

# Inbound authentication (optional). If set, every request must
# present this key via "Authorization: Bearer <key>" or "X-Api-Key: <key>".
# The "/" and "/health" paths always remain accessible (for health-checks).
# Leave it empty/commented to keep the proxy open.
#PROXY_API_KEY=change-this-key

# Largest inbound request body accepted, in MiB. The body is buffered in memory
# before any route sees it, so this bounds what one caller can make a worker
# hold — times the number of threads accepting at once. Raise it if you relay
# genuinely large multimodal prompts; 0 removes the limit. Over the limit the
# proxy answers 413 in the usual OpenAI error shape.
#MAX_REQUEST_MB=32

# Response caching (non-streaming replies only). Identical requests are served
# from an in-memory per-worker cache, skipping the upstream call (saves latency
# and token cost). Streaming requests are never cached. Disabled by default.
# CACHE_ENABLED: 1/true/yes/on to enable. Default false.
# CACHE_TTL: entry time-to-live in seconds. Default 300.
# CACHE_MAX_SIZE: max cached entries (LRU eviction past the cap). Default 512.
# CACHE_POLICY: which requests are eligible. Default deterministic.
#   off           nothing is cached (the counters stay visible at /stats)
#   embeddings    only embeddings, deterministic by construction
#   deterministic embeddings + completions with a seed, or temperature=0 and top_p=1
#   all           every non-streaming 2xx reply: identical prompts return identical
#                 text for the whole TTL, sampling parameters ignored
# Cache activity is reported at /stats and /stats.json (hits, misses, hit rate,
# entries, evictions, expirations, policy skips).
#CACHE_ENABLED=on
#CACHE_TTL=300
#CACHE_MAX_SIZE=512
#CACHE_POLICY=deterministic

# --- Audit trail -------------------------------------------------------------
# One structured record per request (prompt, reply, parameters, provider, tokens,
# timings, session), written to a file by a background thread: the request never
# waits for it, and if the writer falls behind the record is dropped rather than
# slowing the proxy down. Disabled by default. See docs/audit.md.
# NOTE: unless AUDIT_BODIES=none the file contains the conversations in clear
# text — protect it and give it a retention policy.
#AUDIT_ENABLED=on
# Destination file; {pid} and {date} are expanded. Use {pid} with several
# gunicorn workers: they can append to one file but cannot coordinate rotation.
#AUDIT_FILE=logs/audit.jsonl
# jsonl (one record per line, for tail/jq) or pretty (indented, to read directly).
#AUDIT_FORMAT=jsonl
# How much content is recorded: none (accounting only) | truncated | full.
#AUDIT_BODIES=truncated
#AUDIT_MAX_CHARS=2000
# Records that may wait for the writer; past this they are dropped and counted
# at /stats. Raise it if "dropped" is not zero under load.
#AUDIT_QUEUE_SIZE=10000
# Rotation: at most AUDIT_MAX_MB * (AUDIT_BACKUPS + 1) on disk.
#AUDIT_MAX_MB=64
#AUDIT_BACKUPS=5
# Header carrying your front-end's conversation id, if it has one. Without it
# the session is fingerprinted from the conversation's opening message.
#AUDIT_SESSION_HEADER=X-OpenWebUI-Chat-Id

# Embeddings (endpoints /v1/embeddings, /api/embeddings, /api/embed).
# Model used when the client does not specify one (chat models are not valid).
NVIDIA_EMBEDDINGS_MODEL=nvidia/nv-embedqa-e5-v5
# input_type applied if the client does not provide it ("query" or "passage"). Empty to not force it.
EMBEDDINGS_INPUT_TYPE=query

# Single model (backward-compatible). Used as default if NVIDIA_MODELS is not set.
NVIDIA_MODEL=meta/llama-3.1-8b-instruct

# Multi-model: comma-separated list. The first one is the default.
# All listed models appear in Open WebUI (/api/tags and /v1/models).
#NVIDIA_MODELS=meta/llama-3.1-8b-instruct,meta/llama-3.1-70b-instruct,mistralai/mistral-7b-instruct-v0.3
NVIDIA_MODELS=meta/llama-3.2-1b-instruct,nvidia/nemotron-3-nano-omni-30b-a3b-reasoning,nvidia/nemotron-3-nano-30b-a3b,mistralai/mistral-small-4-119b-2603,mistralai/mistral-large-3-675b-instruct-2512,nvidia/nemotron-mini-4b-instruct,nvidia/nemotron-3-content-safety,nvidia/nemotron-content-safety-reasoning-4b,nvidia/nemotron-3.5-content-safety,nvidia/riva-translate-4b-instruct-v1.1,meta/llama-3.1-8b-instruct,nvidia/nemotron-3-ultra-550b-a55b,nvidia/llama-3.1-nemotron-nano-vl-8b-v1,microsoft/phi-4-mini-instruct,upstage/solar-10.7b-instruct,sarvamai/sarvam-m,stockmark/stockmark-2-100b-instruct,nvidia/nemotron-nano-12b-v2-vl,abacusai/dracarys-llama-3.1-70b-instruct,meta/llama-3.2-11b-vision-instruct,google/gemma-3n-e4b-it,mistralai/ministral-14b-instruct-2512,meta/llama-3.2-90b-vision-instruct,mistralai/mixtral-8x7b-instruct-v0.1,nvidia/llama-3.3-nemotron-super-49b-v1