# API
# HTTP API port.
PORT=8191
# Default scrape deadline in milliseconds; per-request maxTimeout takes precedence.
SCRAPE_MAX_TIMEOUT_MS=60000

# Model Context Protocol (MCP) scraping tool. Keep disabled on public networks.
MCP_ENABLED=false
# Comma-separated browser origins allowed to call /mcp. Requests without Origin
# (normal server-to-server clients) are accepted.
MCP_ALLOWED_ORIGINS=

# Optional dashboard. Set ENABLED=true only when its port is bound to 127.0.0.1.
# A token of at least 32 characters protects data if the port is reachable elsewhere.
METRICS_DASHBOARD_ENABLED=false
METRICS_DASHBOARD_TOKEN=
# SQLite history path. Compose mounts a persistent named volume at /data/metrics.
METRICS_DB_PATH=/data/metrics/trawl.sqlite

# Browser pool
# Number of warm browser instances.
BROWSER_POOL_SIZE=1
# Named browser contexts: idle TTL and per-instance capacity (use 1 on small containers).
BROWSER_SESSION_MAX_ENTRIES=4
BROWSER_SESSION_TTL_SECONDS=3600
LOG_LEVEL=info
# Time to wait for a free browser before rejecting a request.
BROWSER_ACQUIRE_TIMEOUT_MS=15000
# Recycle a browser after this many temporary challenge contexts. Set 0 to disable count-based recycling; memory-pressure recovery remains.
BROWSER_RECYCLE_AFTER_CONTEXTS=8
# Maximum Firefox content processes per browser.
BROWSER_MAX_CONTENT_PROCESSES=2
# Set false to omit uBlock Origin and reduce extension overhead; ads/trackers will load.
BROWSER_BLOCK_ADS=true
# Optional idle retirement; 0 keeps browsers warm. Wake adds a cold start.
BROWSER_IDLE_TIMEOUT_MS=0
# Optional: lower the browser-reported CPU count to reduce PoW workers.
# BROWSER_HARDWARE_CONCURRENCY=4
# Browsers in the headful sub-pool, used only for DataDome escalations. Warmed on first use
# and additional to BROWSER_POOL_SIZE (~380 MB each). Set 1 to scrape DataDome targets.
BROWSER_HEADFUL_POOL_SIZE=0
# Lowest scraper tier allowed for every endpoint (1=HTTP, 2=cached browser,
# 3=fresh browser, 4=residential proxy). Values above 1 increase browser load.
SCRAPE_MIN_TIER=1
# Optional Firefox TLS/HTTP2 impersonation in Tier 1: bun (default) or wreq.
TIER1_TRANSPORT=bun
# Optional Firefox prefs applied to every launched browser, as a JSON object.
USER_PREFS=
# Grace period after a request timeout before a checked-out browser is reclaimed.
BROWSER_STALL_TIMEOUT_MS=180000
# Maximum time to wait while closing a wedged browser or context.
BROWSER_CLOSE_TIMEOUT_MS=10000
# Maximum time shutdown waits for accepted API requests. 0 closes them at once.
BROWSER_SHUTDOWN_DRAIN_MS=0
# Maximum time to wait for a browser launch.
BROWSER_LAUNCH_TIMEOUT_MS=90000
# Optional viewport screenshots (browser tiers only)
SCREENSHOT_SETTLE_MS=3000
SCREENSHOT_TIMEOUT_MS=10000
SCREENSHOT_JPEG_QUALITY=60
SCREENSHOT_MAX_BYTES=4000000

# Optional challenge-wall HTML returned on terminal /scrape failures
BLOCKED_EVIDENCE_MAX_HTML_CHARS=512000

# Optional console and network diagnostics (browser tiers only)
DIAGNOSTICS_MAX_CONSOLE_ENTRIES=500
DIAGNOSTICS_MAX_NETWORK_ENTRIES=1000
DIAGNOSTICS_MAX_STRING_CHARS=2000
DIAGNOSTICS_MAX_TOTAL_CHARS=1000000
DIAGNOSTICS_SIZE_TIMEOUT_MS=2000

# Optional main-document redirect chain (browser tiers only)
REDIRECT_MAX_ENTRIES=50
REDIRECT_MAX_URL_CHARS=2000
REDIRECT_MAX_TOTAL_CHARS=1000000

# Optional matched response-body capture (browser tiers only)
CAPTURE_MAX_PATTERNS=10
CAPTURE_MAX_RESPONSES=5
CAPTURE_MAX_BODY_BYTES=5242880
CAPTURE_MAX_TOTAL_BYTES=10485760
CAPTURE_MAX_READ_BYTES=10485760
CAPTURE_MAX_METADATA_CHARS=2000
CAPTURE_BODY_TIMEOUT_MS=5000
CAPTURE_SETTLE_MS=15000
CAPTURE_MAX_SETTLE_MS=60000
CAPTURE_IDLE_FLOOR_MS=5000

# Optional best-effort MHTML archives (browser tiers only)
MHTML_MAX_PARTS=200
MHTML_MAX_PART_BYTES=2097152
MHTML_MAX_TOTAL_CHARS=8388608
MHTML_MAX_INFLIGHT_READS=32
MHTML_MAX_OMISSION_RECORDS=100

# Sessions
# Use "memory" for a bounded, per-process cache with no external dependency.
SESSION_CACHE_DRIVER=redis
# Redis stores solved cookies and browser sessions. Leave empty to disable the Redis cache.
REDIS_URL=redis://localhost:6379
# Lifetime of Redis or in-memory sessions in seconds.
REDIS_SESSION_TTL_SECONDS=3600
# Maximum number of sessions retained by the memory driver (least recently used entries are evicted).
MEMORY_SESSION_CACHE_MAX_ENTRIES=1000
# Maximum time for each Redis connection attempt.
REDIS_CONNECT_TIMEOUT_MS=5000
# Delay between background reconnect attempts. Set 0 to disable retry.
REDIS_RETRY_DELAY_MS=5000

# Upstream proxies
# Pool selection: failover keeps a proxy sticky per domain; roundrobin and random
# choose a new healthy endpoint whenever a request enters Tier 3 or Tier 4.
SCRAPE_PROXY_SELECTION=failover
# Tier 3 datacenter proxy: HTTP, authenticated HTTP, or SOCKS5.
# Examples:
# PROXY_URL=http://proxy.example.com:8080
# PROXY_URL=http://user:pass@proxy.example.com:8080
# PROXY_URL=socks5://proxy.example.com:1080
# Separate multiple endpoints with commas to create a pool.
PROXY_URL=
# Optional file with one proxy URL per line.
PROXY_LIST_FILE=
# Tier 4 residential proxy. Uses the same URL and pool formats.
# Examples:
# RESIDENTIAL_PROXY_URL=http://user:pass@residential.example.com:8080
# RESIDENTIAL_PROXY_URL=socks5://residential.example.com:1080
RESIDENTIAL_PROXY_URL=
# Optional file with one residential proxy URL per line.
RESIDENTIAL_PROXY_LIST_FILE=

# HTTP/HTTPS challenge-bypassing proxy
# Enables the general proxy listener.
MITM_ENABLED=false
# Proxy address and port. Use 127.0.0.1 for local-only access.
MITM_HOST=0.0.0.0
MITM_PORT=8192
# Persistent directory for the generated CA certificate and key.
MITM_CA_DIR=/data/proxy-ca
# Highest allowed escalation tier. Leave empty to allow Tier 4.
MITM_MAX_TIER=
# Skip the direct Tier 0 probe and route requests through the scraper. Disables Tier 0 streaming.
MITM_ALWAYS_SCRAPE=false
# Try the scraper for unrecognized HTTP 429 responses. Can repeat requests and use browsers.
MITM_ESCALATE_429=false
# Logs proxied requests. Errors are always logged.
MITM_DEBUG=false

# CAPTCHA audio
# Optional Whisper-compatible speech-to-text endpoint.
STT_URL=
# Optional API key sent to the speech-to-text endpoint.
STT_API_KEY=
# Optional ffmpeg executable override used for audio conversion.
FFMPEG_PATH=

# Optional paid CAPTCHA fallback. A key alone does not enable it.
# Automatic: single declarative reCAPTCHA v2/Enterprise and standalone Turnstile.
# Other documented task types need hostname profiles with explicit verification.
CAPTCHA_SOLVER=none
TWOCAPTCHA_API_KEY=
# Shared limit across all tiers of one scrape (1-3). No automatic create retries.
CAPTCHA_SOLVER_MAX_TASKS=1
# Both phases remain inside the scrape's maxTimeout.
CAPTCHA_SOLVER_LOCAL_TIMEOUT_MS=10000
CAPTCHA_SOLVER_TIMEOUT_MS=120000
# Optional JSON array of hostname-bound task and delivery profiles. See docs.
CAPTCHA_SOLVER_PROFILES=