"""CashPilot — FastAPI application.

Self-hosted passive income dashboard: service catalog, Docker container
management, and earnings tracking.
"""

from __future__ import annotations

import asyncio
import collections
import contextlib
import hmac
import ipaddress
import json
import logging
import math
import os
import re
import secrets
import socket
import time
import urllib.parse
import weakref
from contextlib import asynccontextmanager
from datetime import UTC, datetime, timedelta
from typing import Any

import httpx
from apscheduler.events import EVENT_JOB_ERROR, EVENT_JOB_MISSED
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from fastapi import Depends, FastAPI, HTTPException, Request, Response
from fastapi.encoders import jsonable_encoder
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, PlainTextResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel, Field, field_validator
from starlette.middleware.base import BaseHTTPMiddleware

from app import (
    activity_ui,
    arch,
    auth,
    catalog,
    compose_generator,
    constants,
    credential_test,
    custom_services,
    database,
    disclosure,
    drift,
    egress,
    exchange_rates,
    fleet_key,
    ipgeo,
    lan_isolation,
    login_rate_limit,
    machine_economics,
    metrics,
    net_activity,
    notify,
    onchain,
    payout_registry,
    payouts,
    placement,
    power,
    preflight,
    producer_state,
    proxies,
    ratings,
    setup_token,
    update_check,
    usage,
    version,
)
from app.collectors.base import REGISTER_PENDING
from app.worker_proxy import _pin_url_to_ip, _validate_worker_url

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger(__name__)

scheduler = AsyncIOScheduler()

# In-memory store for the latest collector alerts (errors from last run)
_collector_alerts: list[dict[str, str]] = []
# Activity alerts of the last day, rebuilt from the alerts table, so the bell
# keeps them across the hourly wholesale rebuild of _collector_alerts.
_activity_bell: list[dict[str, str]] = []
# Whether a collection has ever COMPLETED here. An empty alert list means
# "nothing is wrong" only once something has looked; before that it means
# "nothing has been checked", and the bell used to render both as "All
# collectors healthy" (CashPilot-tb5). Restored on startup from durable state so
# a restart does not reset the claim to "never ran" while data exists.
_collection_has_run: bool = False
# Monotonic time the collection machinery last COMPLETED a run (success or
# failure — this measures aliveness, not outcome; the success timestamp is a
# metric). Initialized to process start so boot gets a grace window, exactly
# like the worker's heartbeat stamp. A dead scheduler, a wedged collection
# lock, or a blocked event loop all freeze this — and they used to freeze
# every other signal green with them.
_last_collection_finished: float = time.monotonic()
_collection_lock = asyncio.Lock()
_collection_semaphore = asyncio.Semaphore(8)

# Fire-and-forget background tasks (e.g. triggered collection runs). Keeping a
# reference prevents the task from being garbage-collected mid-run and lets us
# retrieve/log any exception it raised (bare `asyncio.create_task(...)` drops
# the reference and silently swallows exceptions).
_background_tasks: set[asyncio.Task] = set()


def _spawn(coro) -> asyncio.Task:
    """Fire-and-forget a coroutine while keeping a reference and logging errors."""
    task = asyncio.create_task(coro)
    _background_tasks.add(task)

    def _on_done(t: asyncio.Task) -> None:
        _background_tasks.discard(t)
        if not t.cancelled():
            exc = t.exception()
            if exc is not None:
                logger.error("Background task failed: %s", exc, exc_info=exc)

    task.add_done_callback(_on_done)
    return task


# Login rate limiting moved to app.login_rate_limit (bead sux) — it was the last
# thing the routers genuinely needed from this module, and extracting it is what
# breaks the main -> routers -> main import cycle.
#
# Re-exported by REFERENCE, not copied: tests patch app.main._check_login_rate
# and a conftest fixture clears app.main._login_attempts between tests. Binding a
# copy here would leave both pointing at an object nothing reads, and the fixture
# would silently stop isolating tests.
_login_attempts = login_rate_limit._login_attempts
_LOGIN_MAX_ATTEMPTS = login_rate_limit.MAX_ATTEMPTS
_LOGIN_WINDOW_SECONDS = login_rate_limit.WINDOW_SECONDS
_check_login_rate = login_rate_limit.check_login_rate
_record_failed_login = login_rate_limit.record_failed_login


def _safe_json(raw: str, fallback: Any = None) -> Any:
    """Parse JSON with a fallback so one malformed DB row doesn't 500 the fleet."""
    try:
        return json.loads(raw)
    except (json.JSONDecodeError, TypeError):
        return fallback if fallback is not None else []


def _decoded_worker(worker: dict[str, Any]) -> dict[str, Any]:
    """A worker row with its JSON-TEXT columns decoded into real structures.

    ``list_workers``/``get_worker`` return raw rows, so ``containers`` and
    ``system_info`` arrive as strings. Passing one of those straight to code
    expecting a mapping is an AttributeError at request time, so every caller
    that reads inside them goes through here.
    """
    decoded = dict(worker)
    # Deliberately delegates rather than repeating the three _safe_json calls:
    # a second copy of the decoding is how one caller ends up forgetting it.
    _parse_worker_json(decoded)
    return decoded


# Previous network counter reading per (worker, slug), for turning Docker's
# CUMULATIVE totals into a rate. In memory on purpose: losing it on restart
# costs one unknown reading, whereas persisting a stale baseline across a
# restart risks pairing it with counters that reset in the meantime.
_net_baselines: dict[tuple[Any, str], tuple[int, float]] = {}


def _traffic_state(slug: str, containers: list[dict[str, Any]]) -> str | None:
    """MOVING/SILENT/UNKNOWN for a service, from two counter readings.

    Returns None when there is nothing to say at all, so the caller can leave
    the signal out entirely rather than assert "unknown" about a service whose
    containers were never seen.
    """
    now = time.monotonic()
    rates: list[float] = []
    measured = False

    for container in containers:
        total = net_activity.totals(container)
        if total is None:
            continue
        measured = True
        key = (container.get("_worker_id"), slug)
        previous = _net_baselines.get(key)
        _net_baselines[key] = (total, now)
        if previous is None:
            continue
        value = net_activity.rate(previous[0], total, now - previous[1])
        if value is not None:
            rates.append(value)

    if not measured:
        return None
    if not rates:
        # Counters exist but no usable interval yet — first sight of this
        # container, or its counters reset. Saying "unknown" is the point.
        return net_activity.UNKNOWN
    # Any instance moving data means the service is not silent.
    return net_activity.classify(max(rates))


async def _get_all_worker_containers(workers: list[dict[str, Any]] | None = None) -> list[dict[str, Any]]:
    """Collect container/app data from all online workers' heartbeat data in DB.

    ``workers`` lets a caller that has ALREADY fetched the worker list hand it
    in rather than causing a second `SELECT *` and a second full JSON decode of
    every row in the same request. Everyone else keeps calling this with no
    arguments and gets the fetch for free.
    """
    if workers is None:
        workers = await database.list_workers()
    result: list[dict[str, Any]] = []
    for w in workers:
        if w.get("status") != "online":
            continue
        sys_info = _safe_json(w.get("system_info", "{}"), {})
        worker_has_docker = sys_info.get("docker_available", False)
        is_android = placement.is_android(sys_info)
        worker_name = w.get("name", "worker")

        # Docker containers (from Docker-based workers only — skip for Android)
        if not is_android:
            containers = _safe_json(w.get("containers", "[]"))
            for c in containers:
                slug = c.get("slug", "")
                if slug:
                    result.append(
                        {
                            "slug": slug,
                            "name": c.get("name", slug),
                            "status": c.get("status", "unknown"),
                            "image": c.get("image", ""),
                            "cpu_percent": c.get("cpu_percent", 0),
                            "memory_mb": c.get("memory_mb", 0),
                            "net_rx_bytes": c.get("net_rx_bytes"),
                            "net_tx_bytes": c.get("net_tx_bytes"),
                            "category": "",
                            # The WORKER's answer, not the node name. The image
                            # matcher sets "external" for a container it found by
                            # image rather than by CashPilot's own label — one the
                            # user started themselves. Overwriting it here meant
                            # that container appeared as an ordinary managed
                            # service with live Restart/Stop/Logs buttons, and
                            # every one of them answered "404 Container not
                            # found" for a row the same screen called Running.
                            # The node name is already carried by _node.
                            "deployed_by": c.get("deployed_by") or worker_name,
                            # Which copy on that worker. Without it every copy
                            # reads as #1 and its buttons act on the first one.
                            "instance": c.get("instance") or 1,
                            # The dial-back address the worker read from the ONE
                            # env var the service's catalog entry declares. None
                            # when undeclared/unreported — this dict is a key
                            # allowlist, so without this line the field the
                            # mismatch check depends on would be silently
                            # dropped right here.
                            "advertised_address": c.get("advertised_address"),
                            # When and how a stopped container stopped, for the
                            # container-down alert. Absent on a running one.
                            "finished_at": c.get("finished_at"),
                            "exit_code": c.get("exit_code"),
                            "restart_policy": c.get("restart_policy"),
                            # What the provider said when it refused to run the
                            # client (an account_disabled health signal), read by the
                            # worker from a stopped or crash-looping container.
                            "fatal": c.get("fatal"),
                            # When that refused run ended, to tell a refusal
                            # older than the last balance read (stale) apart.
                            "fatal_at": c.get("fatal_at"),
                            # What the container was created with, for the drift
                            # check (app/drift.py). None from a worker too old to send it.
                            "shape": c.get("shape"),
                            "_arch": sys_info.get("arch"),
                            "_node": worker_name,
                            "_worker_id": w.get("id"),
                            "_has_docker": worker_has_docker,
                            "_is_android": False,
                        }
                    )

        # Android apps (from Android workers)
        if is_android:
            apps = _safe_json(w.get("apps", "[]"))
            for a in apps:
                slug = a.get("slug", "")
                if slug:
                    result.append(
                        {
                            "slug": slug,
                            "name": a.get("slug", slug),
                            # Three-valued, matching what the Android client
                            # now sends. `running` is None when the phone could
                            # not determine it -- every detection signal degrades
                            # to false when its permissions are denied -- and
                            # `"running" if a.get("running") else "stopped"`
                            # turned that into a confident "stopped", so the
                            # fleet page stated the user's earning apps had died
                            # when the device simply could not see them.
                            "status": _android_app_status(a.get("running")),
                            "image": "",
                            "cpu_percent": 0,
                            "memory_mb": 0,
                            "category": "",
                            # Android apps are enumerated by the worker itself and
                            # are never CashPilot-managed containers, so there is
                            # no external/managed distinction to preserve here.
                            "deployed_by": worker_name,
                            "_node": worker_name,
                            "_worker_id": w.get("id"),
                            "_has_docker": False,
                            "_is_android": True,
                            "_net_tx_24h": a.get("net_tx_24h", 0),
                            "_net_rx_24h": a.get("net_rx_24h", 0),
                        }
                    )
    return result


async def _resolve_worker_id(worker_id: int | None) -> int:
    """Return a valid worker_id, auto-resolving when only one worker is online."""
    if worker_id is not None:
        return worker_id
    workers = await database.list_workers()
    online = [w for w in workers if w["status"] == "online"]
    if len(online) == 1:
        return online[0]["id"]
    if len(online) == 0:
        raise HTTPException(status_code=503, detail="No workers online")
    raise HTTPException(
        status_code=400,
        detail="worker_id is required (multiple workers online)",
    )


# ---------------------------------------------------------------------------
# Periodic collection job
# ---------------------------------------------------------------------------


async def _run_health_check() -> None:
    """Check health of all deployed containers and record events.

    Deduplicates by slug: if *any* instance of a service is running,
    record a single check_ok for that slug (avoids penalising services
    deployed on multiple nodes where one may be stopped).

    A service that vanishes from the heartbeat entirely (container removed
    outside CashPilot, crash-removed, worker's Docker daemon down) previously
    got no health event at all — it just stopped appearing, so its score
    stayed frozen wherever it last was (often green) forever. Any known
    Docker-backed deployment missing from every online worker's current data
    now gets an explicit check_down. Scoped to "at least one worker online"
    so a fully-offline fleet (no heartbeat data to trust either way) never
    triggers false check_downs; "external" deployments (e.g. Grass, Bytelixir)
    have no container and are excluded since no worker ever reports them.
    """
    try:
        statuses = await _get_all_worker_containers()
        # Aggregate: slug -> best status (running wins)
        slug_best: dict[str, str] = {}
        for s in statuses:
            slug = s["slug"]
            status = s.get("status", "unknown")
            # A phone that cannot look (no usage or notification access) reports
            # its apps as unknown. That says nothing about the app, so it takes
            # no part here: it neither records downtime on its own nor hides a
            # stopped container another box reports for the same service.
            if s.get("_is_android") and status == "unknown":
                continue
            if slug_best.get(slug) != "running":
                slug_best[slug] = status
        # Collect every event for this cycle and write them in a single transaction
        # rather than one fsync'd commit per service (see database.record_health_events).
        events: list[tuple[str, str, str]] = []
        for slug, status in slug_best.items():
            if status == "running":
                events.append((slug, "check_ok", ""))
            else:
                events.append((slug, "check_down", status))

        workers = await database.list_workers()
        online = [w for w in workers if w.get("status") == "online"]
        # "Missing from a heartbeat" is only evidence of being DOWN if every
        # online worker could actually look. A worker whose Docker socket became
        # unreadable — unmounted by a host update, permissions changed, daemon
        # erroring — keeps heartbeating happily and reports zero containers, so
        # this loop wrote a durable check_down for EVERY deployment every five
        # minutes while those containers were up and earning. The user watched
        # each service's Health column fall and its uptime read 0%, and the
        # events kept dragging the 7-day score long after the socket was fixed.
        #
        # The deployments table has no worker column, so a missing container
        # cannot be attributed to a particular host: if any online worker is
        # blind, the container might be on that one. Unknown is not down, so
        # nothing is recorded — the score is left untouched rather than
        # invented.
        # A phone has no Docker by design, so it cannot be hiding a missing
        # container and is not blind in this sense. Counting it switched the
        # missing-container check off for the whole fleet whenever a phone was
        # online, and warned about it every five minutes.
        # A paired Desktop reports its containers and sends no docker_available
        # either; counting it as blind did the same to every fleet with one.
        blind = [w for w in online if placement.is_blind(w)]
        _report_blind_workers(blind)
        if online and not blind:
            deployments = await database.get_deployments()
            for d in deployments:
                slug = d["slug"]
                if d.get("status") == "external" or slug in slug_best:
                    continue
                events.append((slug, "check_down", "missing from heartbeat"))

        await database.record_health_events(events)
        await _container_down_check(statuses)
        await _account_disabled_check(statuses)
        await _missing_deployment_check(workers)
    except Exception as exc:
        logger.warning("Health check skipped: %s", exc)


# The blind workers the last warning named. The health check runs every five
# minutes, so warning on each run repeated one fact hundreds of times a day; it
# now speaks when the set changes and once when every worker can see again.
_blind_workers_logged: frozenset[str] | None = None


def _report_blind_workers(blind: list[dict[str, Any]]) -> None:
    global _blind_workers_logged
    names = frozenset(str(w.get("name") or w.get("id")) for w in blind)
    if names == (_blind_workers_logged or frozenset()):
        return
    _blind_workers_logged = names
    if names:
        logger.warning(
            "Not recording missing-container downtime: %d online worker(s) cannot read Docker (%s). "
            "Their containers may be running; recording them as down would be a guess.",
            len(names),
            ", ".join(sorted(names)),
        )
    else:
        logger.info("Every online worker can read Docker again; missing-container downtime is recorded again.")


# How long a managed container may stay exited before it is an alert. Long
# enough for a deploy, an update or a manual restart to finish; short enough
# that a node which exited cleanly and stayed down (restart policy "no") is
# news within the hour instead of after days of check_down rows nobody reads.
CONTAINER_DOWN_GRACE = timedelta(minutes=30)
# A stop recorded by CashPilot explains a container that finished within this
# long after it; a container that finished later stopped for another reason.
_STOP_EXPLAINS_WITHIN = timedelta(minutes=10)
# When each stopped container was first seen down, for a worker too old to
# report Docker's own FinishedAt. Keyed by (worker id, container name).
_container_first_seen_down: dict[tuple[Any, str], datetime] = {}
# The bell entries the check currently stands behind, re-added after every
# collection, which replaces the bell list wholesale.
_container_down_bell: dict[str, dict[str, str]] = {}
_RESTARTING_POLICIES = ("unless-stopped", "always")


def _docker_time(value: Any) -> datetime | None:
    """Docker's FinishedAt (RFC 3339 in UTC, nanoseconds) as a datetime."""
    if not isinstance(value, str) or len(value) < 19:
        return None
    try:
        return datetime.fromisoformat(value[:19]).replace(tzinfo=UTC)
    except ValueError:
        return None


async def _container_down_check(statuses: list[dict[str, Any]]) -> None:
    """Alert when a CashPilot-managed container has stayed exited past the grace period.

    The health check already wrote check_down for such a container every five
    minutes, and nothing read it: three urnetwork containers exited cleanly
    with restart policy "no" and sat down for two days with no alert. One alert
    per service, cleared when no managed copy of it is down any more. A stop the
    user made from CashPilot is not an alert. Never raises.
    """
    global _collector_alerts
    try:
        now = datetime.now(UTC)
        down: dict[str, list[dict[str, Any]]] = {}
        seen_keys: set[tuple[Any, str]] = set()
        for s in statuses:
            if s.get("_is_android") or s.get("deployed_by") == "external":
                continue
            if s.get("status") not in ("exited", "dead"):
                continue
            # The provider refused it (account disabled): starting it again
            # cannot help, and the account-disabled alert says so instead.
            if s.get("fatal"):
                continue
            key = (s.get("_worker_id"), str(s.get("name") or s.get("slug")))
            seen_keys.add(key)
            since = _docker_time(s.get("finished_at")) or _container_first_seen_down.setdefault(key, now)
            down.setdefault(s["slug"], []).append({**s, "_down_since": since})
        for key in set(_container_first_seen_down) - seen_keys:
            _container_first_seen_down.pop(key, None)

        alerting: dict[str, str] = {}
        for slug, copies in down.items():
            long_down = [c for c in copies if now - c["_down_since"] >= CONTAINER_DOWN_GRACE]
            if not long_down:
                continue
            last = await database.get_last_lifecycle_event(slug)
            if last and last[0] in ("stop", "remove"):
                stopped_at = _docker_time(last[1].replace(" ", "T"))
                if stopped_at is not None:
                    long_down = [c for c in long_down if c["_down_since"] > stopped_at + _STOP_EXPLAINS_WITHIN]
            if long_down:
                alerting[slug] = _container_down_message(slug, long_down)

        # Recovery: nothing of that service is down any more. A copy that is
        # down but still inside the grace period (or explained by a stop) is
        # neither raised nor cleared, so a UI restart cannot clear an alert
        # just because it forgot when an old worker's container went down.
        for slug in await database.get_alert_subjects("container"):
            if slug not in down:
                await database.clear_alerts("container", slug)
                logger.info("%s is running again; cleared its container-down alert", slug)
        for slug in list(_container_down_bell):
            if slug not in alerting:
                _container_down_bell.pop(slug)

        for slug, message in alerting.items():
            _container_down_bell[slug] = {"kind": "container", "platform": slug, "error": message}
            if alert_id := await database.record_alert("container", slug, message):
                _spawn(_push_alert("container", slug, f"CashPilot: {slug} has stopped running", message, alert_id))

        _collector_alerts = [a for a in _collector_alerts if a.get("kind") != "container"]
        _collector_alerts.extend(_container_down_bell.values())
    except Exception as exc:  # noqa: BLE001 - a diagnostic must not break the health check
        logger.warning("Container-down check failed: %s", exc)


def _container_down_message(slug: str, copies: list[dict[str, Any]]) -> str:
    name = (catalog.get_service(slug) or {}).get("name") or slug
    since = min(c["_down_since"] for c in copies).strftime("%Y-%m-%d %H:%M")
    nodes = ", ".join(sorted({str(c.get("_node") or "?") for c in copies}))
    codes = sorted({c["exit_code"] for c in copies if isinstance(c.get("exit_code"), int)})
    how = f" (exit code {', '.join(str(code) for code in codes)})" if codes else ""
    message = f"{name} has not been running since {since} UTC on {nodes}{how}, so it is not earning."
    if catalog.fence_gated(catalog.get_service(slug)):
        # No restart policy on purpose: the worker starts it once the LAN fence is confirmed.
        return message + (
            " The worker starts it as soon as the LAN fence is confirmed; while it stays down, the Fleet page "
            "says what holds it. If you stopped it on purpose, start it from the dashboard when you want it back."
        )
    policies = sorted({str(c["restart_policy"]) for c in copies if c.get("restart_policy")} - set(_RESTARTING_POLICIES))
    if policies:
        message += (
            f' Its restart policy is "{", ".join(policies)}", so Docker will not start it again by itself;'
            " redeploying it from CashPilot sets unless-stopped."
        )
    return message + " Start it from the dashboard, or remove it if you stopped it on purpose."


# Services whose provider has disabled the account: slug -> the message shown.
# One alert per service, whatever number of boxes crash-loop on it.
_account_disabled: dict[str, str] = {}
# Platforms whose collector said, on its last run, that the account is disabled.
_collector_says_disabled: set[str] = set()
# Platforms the containers' logs said were disabled on the last check that had
# heartbeat data. Kept apart so a check without data keeps only what the logs
# said, not a collector-only entry the collection loop has since dropped.
_disabled_by_logs: set[str] = set()
# Platforms whose collector read a balance on its last run. A balance read means
# the account works again, whatever an old container log still says.
_collected_ok_last_run: set[str] = set()
# When the collector last read a balance for a service that declares an
# account_disabled signal, kept in the config table so it survives a restart.
# A refusal from a run that ended before it is history: a stopped copy keeps
# the old refusal in its log after the account is restored, and without this a
# later 429 or UI restart raised the alert and pushed again.
_BALANCE_READ_KEY = "_balance_read_at:"
# The alert stays one alert for as long as the account stays disabled: it is
# cleared, and so armed again, only when the account comes back.
_ACCOUNT_DISABLED_COOLDOWN_HOURS = 24 * 365


def _account_disabled_message(slug: str) -> str:
    name = (catalog.get_service(slug) or {}).get("name") or slug
    return (
        f"{name} has disabled this account. Restarting will not help: only {name} can turn it back on, "
        f"so contact their support. CashPilot leaves its containers as they are."
    )


def _declares_account_disabled(slug: str) -> bool:
    signals = ((catalog.get_service(slug) or {}).get("docker") or {}).get("health_signals")
    return isinstance(signals, list) and any(
        isinstance(s, dict) and s.get("state") == catalog.ACCOUNT_DISABLED for s in signals
    )


async def _record_balance_reads(collected_ok: set[str]) -> None:
    """Remember when a balance was read, for the services whose refusal the worker can report."""
    now = datetime.now(UTC).isoformat()
    for slug in sorted(collected_ok):
        if _declares_account_disabled(slug):
            try:
                await database.set_config(_BALANCE_READ_KEY + slug, now)
            except Exception as exc:  # noqa: BLE001 - losing the stamp only keeps an alert up
                logger.warning("Could not record the %s balance read: %s", slug, exc)


async def _balance_read_at(slug: str) -> datetime | None:
    try:
        raw = await database.get_config(_BALANCE_READ_KEY + slug)
        return datetime.fromisoformat(raw) if isinstance(raw, str) and raw else None
    except Exception:  # noqa: BLE001 - unknown means no read to compare against
        return None


async def _account_disabled_check(statuses: list[dict[str, Any]]) -> None:
    """Raise one alert per service whose provider disabled the account; clear it when the account is back.

    Two sources say so: a worker that matched an account_disabled health signal
    in a crash-looping or stopped container, and a collector whose login was
    refused as a disabled account. A balance read beats any refusal from a run
    that ended before it, also across restarts. Never stops or removes
    anything. Never raises.
    """
    global _collector_alerts, _disabled_by_logs
    try:
        if statuses:
            from_logs: set[str] = set()
            read_at: dict[str, datetime | None] = {}
            for s in statuses:
                if s.get("fatal") != catalog.ACCOUNT_DISABLED:
                    continue
                slug = s["slug"]
                if slug not in read_at:
                    read_at[slug] = await _balance_read_at(slug)
                refused_at = _docker_time(s.get("fatal_at"))
                if read_at[slug] and refused_at and refused_at <= read_at[slug]:
                    continue
                from_logs.add(slug)
        else:
            # No heartbeat data at all says nothing about the containers: keep
            # what they said last.
            from_logs = set(_disabled_by_logs)
        _disabled_by_logs = set(from_logs)
        from_logs -= _collected_ok_last_run
        disabled = from_logs | _collector_says_disabled
        for slug in list(_account_disabled):
            if slug not in disabled:
                _account_disabled.pop(slug)
                await database.clear_alerts("account_disabled", slug)
                logger.info("%s: the provider no longer refuses the account; cleared its alert", slug)
        for slug in sorted(disabled):
            message = _account_disabled_message(slug)
            _account_disabled[slug] = message
            if alert_id := await database.record_alert(
                "account_disabled", slug, message, cooldown_hours=_ACCOUNT_DISABLED_COOLDOWN_HOURS
            ):
                name = (catalog.get_service(slug) or {}).get("name") or slug
                _spawn(
                    _push_alert("account_disabled", slug, f"CashPilot: {name} disabled your account", message, alert_id)
                )
        _collector_alerts = _with_account_disabled(_collector_alerts)
    except Exception as exc:  # noqa: BLE001 - a diagnostic must not break the health check
        logger.warning("Account-disabled check failed: %s", exc)


def _with_account_disabled(alerts: list[dict[str, Any]]) -> list[dict[str, Any]]:
    """The bell list with one account-disabled entry per such service in place of its symptoms.

    A disabled account also fails the collector (often as a rate limit, from the
    crash loop's logins) and stops the container; those entries point at the
    wrong fixes, so they give way to the one that names the cause.
    """
    kept = [
        a
        for a in alerts
        if a.get("kind") != "account_disabled"
        and not (a.get("platform") in _account_disabled and a.get("kind", "collector") in ("collector", "container"))
    ]
    kept.extend({"kind": "account_disabled", "platform": slug, "error": msg} for slug, msg in _account_disabled.items())
    return kept


# When each recorded deployment was first seen missing from every heartbeat
# (key: slug and worker id, or None for "no box at all"). See placement.missing.
_deployment_first_missing: dict[tuple[str, Any], datetime] = {}
# The bell entries the missing-deployment check stands behind, re-added after
# every collection like the container-down ones.
_missing_bell: dict[str, dict[str, Any]] = {}


async def _missing_deployment_check(workers: list[dict[str, Any]] | None = None) -> None:
    """Alert when CashPilot records a deployment that no worker runs any more.

    The deployments table outlives a container removed by hand or a rebuilt box,
    and every screen that read it went on calling the service deployed. One
    alert per service, after placement.MISSING_GRACE, through the usual
    push path; it clears when a worker reports the service again, or when the
    record is forgotten (DELETE /api/deployments/{slug}). When the fleet
    cannot tell (no worker online, or a Docker host that cannot read its
    socket), nothing is raised or cleared. Never raises.
    """
    global _collector_alerts
    try:
        if workers is None:
            workers = await database.list_workers()
        stored = await database.get_alert_subjects("missing")
        scopes = {slug: _missing_scope(slug) for slug in stored}
        found = placement.missing(
            await database.get_deployments(),
            await database.list_deployment_placements(),
            workers,
            now=datetime.now(UTC),
            first_missing=_deployment_first_missing,
            alerted=set().union(*scopes.values()) if scopes else set(),
        )
        if found is None:
            return
        by_id = {w.get("id"): w for w in workers}
        where = placement.runs_on(workers)
        for slug in set(stored) | set(_missing_bell):
            if slug in found:
                continue
            # What the alert covered and is still absent, or sits on a box that
            # cannot be looked at right now: that much of it stands.
            scope = scopes.get(slug) or _missing_scope(slug)
            left = [wid for s, wid in _deployment_first_missing if s == slug and ((s, wid) in scope or s in scope)]
            if not left:
                if slug in stored:
                    await database.clear_alerts("missing", slug)
                    logger.info("%s is no longer recorded on a box that does not run it; cleared its alert", slug)
                _missing_bell.pop(slug, None)
                continue
            entry = _missing_bell.get(slug)
            if entry is not None and "everywhere" in entry:
                kept = {
                    "everywhere": None in left,
                    "boxes": [
                        {"worker_id": wid, "name": (by_id.get(wid) or {}).get("name") or str(wid)}
                        for wid in sorted(w for w in left if w is not None)
                    ],
                    "still_on": [r["name"] for r in where.get(slug, [])],
                }
                entry.update(
                    everywhere=kept["everywhere"],
                    boxes=kept["boxes"],
                    error=_missing_deployment_message(slug, kept),
                )
        for slug, item in found.items():
            message = _missing_deployment_message(slug, item)
            _missing_bell[slug] = {
                "kind": "missing",
                "platform": slug,
                "name": (catalog.get_service(slug) or {}).get("name") or slug,
                "error": message,
                # What Forget acts on: the whole record, or one box's.
                "everywhere": item["everywhere"],
                "boxes": item["boxes"],
            }
            if alert_id := await database.record_alert("missing", slug, message):
                name = (catalog.get_service(slug) or {}).get("name") or slug
                title = f"CashPilot: {name} " + ("runs on no box" if item["everywhere"] else "is gone from a box")
                _spawn(_push_alert("missing", slug, title, message, alert_id))
        _collector_alerts = [a for a in _collector_alerts if a.get("kind") != "missing"]
        _collector_alerts.extend(_missing_bell.values())
    except Exception as exc:  # noqa: BLE001 - a diagnostic must not break the health check
        logger.warning("Missing-deployment check failed: %s", exc)


def _missing_scope(slug: str) -> set[Any]:
    """What a stored missing alert of ``slug`` covers, as placement.missing keys.

    A bare slug means the whole service: an alert restored at startup, whose
    boxes are not known until the next check names them.
    """
    entry = _missing_bell.get(slug)
    if entry is None or "everywhere" not in entry:
        return {slug}
    keys: set[Any] = {(slug, b["worker_id"]) for b in entry.get("boxes") or []}
    if entry["everywhere"]:
        keys.add((slug, None))
    return keys


def _missing_deployment_message(slug: str, item: dict[str, Any]) -> str:
    name = (catalog.get_service(slug) or {}).get("name") or slug
    boxes = ", ".join(b["name"] for b in item["boxes"])
    if item["everywhere"]:
        where = f" (it was deployed on {boxes})" if boxes else ""
        return (
            f"CashPilot records {name} as deployed{where}, but no box runs it, so it is not earning. "
            "Deploy it again, or forget the deployment if you removed it on purpose."
        )
    still = ", ".join(item["still_on"])
    return (
        f"{name} was deployed on {boxes}, but {'that box no longer runs' if len(item['boxes']) == 1 else 'those boxes no longer run'} it"
        f"{f' (it still runs on {still})' if still else ''}. "
        "Deploy it there again, or forget it on that box if you removed it on purpose."
    )


async def _push_alert(kind: str, subject: str, title: str, message: str, alert_id: int) -> None:
    """Push an alert out-of-band, and make a total delivery failure retryable.

    The dedupe row is committed BEFORE this runs, so a failed delivery used to
    disarm the alert for the whole cooldown window: ntfy unreachable in the
    minute a collector broke meant one WARNING in a container log and 24 silent
    hours — the push path's own failure was the one failure it could not
    report. When no target accepts the message, the row THIS delivery owns
    (record_alert's returned id) is deleted so the next collection cycle
    records-and-pushes again — by id, never by (kind, subject), because a
    recovery and a fresh failure can insert a newer row while this push was in
    flight, and taking that row's dedupe with ours would double-notify. The
    outcome feeds a metric either way so dead delivery is visible to
    Prometheus instead of indistinguishable from health.
    """
    try:
        delivered = await notify.send(title, message, kind=kind, subject=subject)
    except Exception as exc:  # noqa: BLE001 - a notifier crash must never kill collection
        logger.warning("Alert push for %s/%s crashed: %s", kind, subject, notify.redact(str(exc)))
        delivered = 0
    if not notify.is_enabled():
        # Nothing is configured: the bell is the only channel, the row must
        # stand (clearing it would re-insert every cycle and churn the bell),
        # and delivery metrics would only count an absence.
        return
    metrics.record_notify_delivery(bool(delivered))
    if not delivered:
        # The worker push floor is armed at spawn time, before the outcome is
        # known — a TOTAL delivery failure must release it, or the un-dedupe
        # retry below would be suppressed for the whole floor window and the
        # "will retry on the next cycle" promise becomes false.
        if kind == "worker":
            _worker_last_offline_push.pop(subject, None)
        try:
            undeduped = await database.delete_alert(alert_id)
        except Exception as exc:  # noqa: BLE001
            undeduped = False
            logger.warning("Could not un-dedupe alert %s after failed delivery: %s", alert_id, exc)
        if undeduped:
            logger.warning(
                "No notification target accepted the %s alert for %s — will retry on the next cycle",
                kind,
                subject,
            )
        else:
            # Saying "will retry" when the dedupe row survived would be false.
            logger.warning(
                "No notification target accepted the %s alert for %s, and the dedupe row could "
                "not be released — the next push waits out the cooldown window",
                kind,
                subject,
            )


async def _detect_payout(result: Any) -> dict[str, str] | None:
    """Notice a balance drop that looks like a cashout, and ask.

    Never records income on its own. A balance also falls for a provider
    correction or a reset session, and an unconfirmed guess written as earnings
    corrupts lifetime-earned permanently and invisibly.

    Returns the alert to show the user, or None. It has to be RETURNED rather
    than filed and forgotten: the bell renders one in-memory list built during
    the collection run, and this function runs in the success branch that never
    touched that list — so a detected payout was written to the database and
    then never shown to anybody, which defeats the point of asking.
    """
    try:
        previous = await database.get_latest_balance(result.platform)
        if previous is None:
            return None
        try:
            withdrawn = await database.withdraw_recovered_payout(result.platform, result.balance - previous)
        except Exception as exc:  # noqa: BLE001 - must never block detecting a new payout
            withdrawn = False
            logger.warning("Could not check %s for a recovered payout: %s", result.platform, exc)
        if withdrawn:
            # The drop was one bad reading, not a cashout: the question is taken back.
            logger.info("%s balance recovered; withdrew the probable payout question", result.platform)
            return None
        probable = payouts.detect(previous, result.balance, catalog.get_service(result.platform))
        if not probable:
            return None
        payout_id = await database.record_probable_payout(
            platform=result.platform,
            amount=probable["amount"],
            currency=result.currency,
            fx_rate_usd=exchange_rates.to_usd(1.0, result.currency),
        )
        if payout_id is None:
            # One already pending for this platform; a second prompt for the
            # same event teaches the user to dismiss them.
            return None
        if alert_id := await database.record_alert("payout", result.platform, probable["reason"]):
            # The one alert that asks a QUESTION — unanswered, the payout never
            # counts toward lifetime earnings, so the headline number stays
            # wrong until a human replies. It was the only pushable kind that
            # never pushed: recorded, belled, and then waiting for someone to
            # happen to open the UI. The record_alert gate was already here,
            # its boolean just went unread.
            _spawn(
                _push_alert(
                    "payout",
                    result.platform,
                    f"CashPilot: {result.platform} balance dropped — was this a payout?",
                    probable["reason"],
                    alert_id,
                )
            )
        return {"kind": "payout", "platform": result.platform, "error": probable["reason"]}
    except Exception as exc:
        # Earnings collection must not fail because payout detection did.
        logger.warning("Payout detection failed for %s: %s", getattr(result, "platform", "?"), exc)
    return None


async def _pending_payout_alerts(seen: set[str] | None = None) -> list[dict[str, str]]:
    """One alert per payout still waiting for a yes or no.

    The question outlives the collection run that noticed it. Rebuilding the
    bell only from what was just detected made the prompt disappear on the next
    run, which is the one thing a prompt must not do — an unanswered payout is
    unanswered until the user answers it.
    """
    seen = seen or set()
    try:
        rows = await database.get_payouts()
    except Exception as exc:
        logger.warning("Could not load pending payouts for the alert bell: %s", exc)
        return []
    pending: list[dict[str, str]] = []
    for row in rows:
        platform = row.get("platform", "")
        if row.get("confirmed") or platform in seen:
            continue
        seen.add(platform)
        amount = row.get("amount")
        currency = row.get("currency") or ""
        pending.append(
            {
                "kind": "payout",
                "platform": platform,
                "error": f"Balance dropped by {amount:.2f} {currency}".rstrip()
                + " — was this a payout? Confirm or reject it."
                if isinstance(amount, int | float)
                else "A balance drop looks like a payout — confirm or reject it.",
            }
        )
    return pending


async def _collect_bounded(collector) -> Any:
    """Run a single collector's `collect()` under the shared concurrency limit.

    A raised exception is converted into an EarningsResult HERE, while the
    collector -- and with it the platform name -- is still in hand. It used to
    propagate to the gather(), where the platform was unrecoverable: the
    failure was logged as an anonymous line and produced no alert, no bell
    entry and no metric. A collector that raised was invisible to the user
    (CashPilot-5bdm).
    """
    from app.collectors import base as collectors_base

    async with _collection_semaphore:
        try:
            return await collector.collect()
        except Exception as exc:
            collectors_base.log_failure(logger, collector.platform or type(collector).__name__, exc)
            return collectors_base.EarningsResult(
                platform=collector.platform or type(collector).__name__,
                balance=0.0,
                error=collectors_base.describe_exception(exc),
                error_kind=collectors_base.classify_exception(exc),
            )


async def _offline_worker_alerts() -> list[dict[str, str]]:
    """Bell entries for every worker currently marked offline.

    Derived from the workers table on each rebuild rather than carried in
    memory: the collection loop replaces _collector_alerts wholesale every
    hour, and an entry that lived only in the list would be dropped there —
    silently un-reporting a machine that is still down. Recovery needs no
    bookkeeping either: the next heartbeat flips the row back to online and
    the entry stops being derived.
    """
    try:
        workers = await database.list_workers()
    except Exception as exc:  # noqa: BLE001 - the bell must not take down collection
        logger.warning("Could not derive offline-worker alerts: %s", exc)
        return []
    entries: list[dict[str, str]] = []
    for w in workers:
        if w.get("status") == "offline" and w.get("last_heartbeat"):
            entries.append(
                {
                    "kind": "worker",
                    "platform": w["name"],
                    # Identity travels beside the display name: recovery and
                    # dedupe match on client_id, because two hosts can share a
                    # hostname and hostnames churn across recreates.
                    "client_id": w.get("client_id") or w["name"],
                    "error": (
                        f"Offline since {w['last_heartbeat']} UTC. Its containers keep running "
                        "and earning, but CashPilot cannot see or manage them until it reconnects."
                    ),
                }
            )
    return entries


def _quiet_days_by_platform() -> dict[str, int]:
    """Each catalog entry's collector.quiet_days, for the services that declare one."""
    out: dict[str, int] = {}
    for svc in catalog.get_services():
        raw = (svc.get("collector") or {}).get("quiet_days")
        with contextlib.suppress(TypeError, ValueError):
            if raw is not None and int(raw) > 0:
                out[svc["slug"]] = int(raw)
    return out


def _flatline_message(flat: dict[str, Any]) -> str:
    name = (catalog.get_service(flat["platform"]) or {}).get("name") or flat["platform"]
    days = flat.get("days_flat")
    span = f" ({days} days)" if days else ""
    if flat.get("last_increase") is None:
        return (
            f"{name} has not earned anything since CashPilot started reading it"
            f"{' on ' + str(flat['since']) if flat.get('since') else ''}{span}. "
            "Common causes: the device is not linked to your account, the provider has no demand for your "
            "IP or region yet, or the node cannot be reached from outside. "
            "This clears by itself when the balance grows."
        )
    gap = flat.get("usual_gap_days")
    rhythm = (
        f"it usually earns every {'day' if gap == 1 else f'{gap} days'}"
        if gap
        else "it earned before, though too rarely to say how often"
    )
    return (
        f"{name} hasn't earned since {flat['last_increase']}{span}; {rhythm}. "
        "Common causes: no demand for your IP or region right now, the device was unlinked from your "
        "account, or the node cannot be reached from outside. Services often go quiet for a while and "
        "pick up again; this clears by itself when the balance grows."
    )


# One push per quiet spell. The default 24 h cooldown would repeat the push
# every day the balance stays flat; the alert row is cleared when the balance
# grows, and that is what re-arms it.
_FLATLINE_COOLDOWN_HOURS = 24 * 365 * 10


async def _flatline_check(
    collected: set[str] | None = None,
    deployments: list[dict[str, Any]] | None = None,
    statuses: list[dict[str, Any]] | None = None,
    unlinked: set[str] | None = None,
) -> list[dict[str, str]]:
    """Give a heads-up on a service whose balance has stopped growing.

    A container can be up and a collector can authenticate happily while the
    balance never moves. Every other view of the system looks healthy, so
    nothing else would surface it.

    Soft evidence, so it is held back unless everything else is fine: the
    collector succeeded this run (``collected``), and a service with a container
    has a copy running on an online worker (``statuses`` only lists online
    workers). A container that is down has its own alert, and an offline worker
    its own. When a precondition fails the alert is neither raised nor cleared;
    it clears only when the balance grows, so a payout or a restart cannot
    re-arm it into a second push.

    Never raises: this is a diagnostic, and a diagnostic must not be able to
    take down the collection run it is diagnosing.
    """
    bell: list[dict[str, str]] = []
    try:
        first_deployed = {d["slug"]: d.get("deployed_at") or "" for d in deployments or []}
        flat_services = await database.get_flatlined_services(
            quiet_days=_quiet_days_by_platform(), first_deployed=first_deployed
        )
        flat_now = {f["platform"] for f in flat_services}

        # Clear the row for anything that has started earning again, so its
        # next quiet spell notifies instead of being deduped into silence.
        for recovered in await database.get_alert_subjects("flatline") - flat_now:
            await database.clear_alerts("flatline", recovered)
            logger.info("%s is earning again; cleared its flatline alert", recovered)

        has_container = {d["slug"] for d in deployments or [] if d.get("status") != "external"}
        running = {s["slug"] for s in statuses or [] if s.get("status") == "running"}
        for flat in flat_services:
            platform = flat["platform"]
            if collected is not None and platform not in collected:
                continue
            # A collector found a deployed device that is not on the account,
            # and its notice already names that cause; a vaguer heads-up for
            # the same service would only repeat it.
            if unlinked and platform in unlinked:
                continue
            if statuses is not None and platform in has_container and platform not in running:
                continue
            message = _flatline_message(flat)
            # Built for EVERY flatlined service, not only the ones that clear
            # record_alert's cooldown. The cooldown exists to stop repeat
            # NOTIFICATIONS; the bell is a standing statement of what is true
            # right now.
            bell.append({"kind": "flatline", "platform": platform, "error": message})
            if alert_id := await database.record_alert(
                "flatline", platform, message, cooldown_hours=_FLATLINE_COOLDOWN_HOURS
            ):
                name = (catalog.get_service(platform) or {}).get("name") or platform
                _spawn(_push_alert("flatline", platform, f"CashPilot: {name} has gone quiet", message, alert_id))
    except Exception as exc:
        logger.warning("Flatline check failed: %s", exc)
        # The check's own failure must not render as health: an empty return
        # here is indistinguishable from "nothing is flatlined", so a broken
        # query would silently retire the only detector this class of fault has.
        bell.append(
            {
                "kind": "flatline",
                "platform": "flatline check",
                "error": "The flatline check itself failed — earning anomalies are NOT being watched. See server logs.",
            }
        )
    return bell


async def _warm_collector_alerts() -> None:
    """Restore persisted collector alerts into the in-memory list the UI bell reads.

    Without this, a restart clears the bell while the collector is still broken, and
    the operator is told everything is fine until the next hourly run.
    """
    global _collector_alerts
    try:
        stored = await database.list_alerts(limit=100)
    except Exception as exc:
        logger.warning("Could not restore persisted alerts: %s", exc)
        return
    # Newest-first, keeping only the most recent row per subject: a normal run puts
    # exactly one entry per failing platform in this list, and the restored bell must
    # look the same rather than showing one row per historical message.
    seen: set[str] = set()
    restored: list[dict[str, str]] = []
    for alert in stored:
        # Payouts belong here too. Dropping every non-collector kind meant a
        # detected payout was recorded and then never shown, so the prompt the
        # user is supposed to answer never appeared. Dedup by kind AND subject:
        # a platform can legitimately have both a failing collector and a
        # pending payout, and they are different things to tell someone.
        # flatline belongs here for the same reason payout does: it is recorded,
        # and without this it is dropped on the way to the bell, so a restart
        # silently clears a warning about a service that is still not earning.
        # worker likewise: an offline machine must survive a UI restart. And
        # notice: #326 made notices durable and deduped, but a restart dropped
        # them from the bell while their 24h push-dedupe survived — so a
        # recovery inside the gap skipped clear_alerts and the stale row
        # swallowed the next genuine warning.
        if alert["kind"] not in (
            "collector",
            "payout",
            "flatline",
            "worker",
            "notice",
            "container",
            "missing",
            "account_disabled",
        ):
            continue
        key = f"{alert['kind']}:{alert['subject']}"
        if key in seen:
            continue
        seen.add(key)
        entry = {"kind": alert["kind"], "platform": alert["subject"], "error": alert["message"]}
        if alert.get("category"):
            entry["category"] = alert["category"]
        restored.append(entry)
    # Worker alerts are stored under client_id (the identity); the bell shows
    # people a NAME. Resolve display names in one pass, keeping the client_id
    # beside it for recovery matching. A worker deleted since the alert was
    # stored keeps the raw id — still actionable, never wrong.
    worker_entries = [e for e in restored if e["kind"] == "worker"]
    if worker_entries:
        with contextlib.suppress(Exception):
            names = {w.get("client_id"): w["name"] for w in await database.list_workers()}
            for e in worker_entries:
                e["client_id"] = e["platform"]
                e["platform"] = names.get(e["platform"], e["platform"])
    for e in restored:
        if e["kind"] == "account_disabled":
            _account_disabled.setdefault(e["platform"], e["error"])
            # Which source raised it is not stored; until a heartbeat says,
            # treat it as the logs' so a check without data keeps it.
            _disabled_by_logs.add(e["platform"])
    # A capped proxy is restored from the proxies table, which says whether the
    # cap still holds; the alerts table only says it once did.
    restored.extend(await _proxy_cap_alerts())
    _collector_alerts = _with_account_disabled(restored)
    # The collection rebuild re-adds _missing_bell, not the restored list, so a
    # restored missing alert goes there too, with no scope: until the next
    # health check names its boxes, the bell offers no Forget for it.
    for e in restored:
        if e["kind"] == "missing":
            name = (catalog.get_service(e["platform"]) or {}).get("name") or e["platform"]
            _missing_bell.setdefault(e["platform"], {**e, "name": name})
    # A restart must not make the bell claim nothing has ever been checked. Any
    # stored alert, or any earnings row, is proof that a collection ran.
    global _collection_has_run
    if restored:
        _collection_has_run = True
    else:
        try:
            _collection_has_run = bool(await database.get_earnings_summary())
        except Exception as exc:  # noqa: BLE001 - a warm-up must not block startup
            logger.warning("Could not tell whether a collection has run before: %s", exc)


async def _track_fully_configured_services() -> int:
    """Give every fully-credentialled service a deployment row, so it is collected.

    Collection iterates DEPLOYMENT ROWS (``make_collectors``), so a service with
    no row is never collected however complete its credentials are. Saving
    credentials has created that row since the fix in #187 — but only at save
    time, and nothing has ever repaired a database written before it.

    That leaves the upgrade path in exactly the state the fix was meant to end:
    credentials stored under an older version show the green "Configured" badge,
    no collector is ever built for them, and no earnings arrive. Nothing prompts
    the user to re-save, because as far as the UI is concerned everything is
    already correct. So this runs once per start and is idempotent — a slug that
    already has a row of any status is left exactly as it is, because that row
    may carry a real container id and spec that must not be replaced with an
    empty placeholder.

    Returns the number of rows created, for the log line and the tests.
    """
    from app.collectors import fully_configured_slugs

    try:
        config = await database.get_config() or {}
        if not isinstance(config, dict):
            return 0
        existing = {d.get("slug") for d in await database.get_deployments()}
        # Removed from the whole fleet by the user: saved credentials are not a
        # reason to bring it back. Deploying it, or saving a changed value or a
        # retyped secret for it in Settings, lifts this (api_set_config,
        # database.save_deployment).
        removed = await database.removed_slugs()
        tracked = 0
        for slug in sorted(fully_configured_slugs(config)):
            if slug in existing or slug in removed or not catalog.get_service(slug):
                continue
            await database.save_deployment(slug=slug, container_id="", status="external")
            tracked += 1
        if tracked:
            logger.info(
                "Started tracking %d service(s) whose credentials were stored before "
                "saving them began tracking; their earnings will be collected from now on",
                tracked,
            )
        return tracked
    except Exception as exc:  # noqa: BLE001 - bookkeeping must never block startup
        logger.warning("Could not backfill tracking for stored credentials: %s", exc)
        return 0


#: A pending registration is retried this long after the first refusal, then
#: twice as long each time, up to the cap; after the give-up it is the user's.
DEVICE_LINK_FIRST_RETRY = timedelta(minutes=6)
DEVICE_LINK_MAX_RETRY = timedelta(hours=4)
DEVICE_LINK_GIVE_UP = timedelta(hours=24)


def _db_time(when: datetime) -> str:
    """The format SQLite's datetime('now') writes, so stored times compare as text."""
    return when.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S")


def _device_name(device_id: str, env: dict[str, Any]) -> str:
    name = str(env.get("DEVICE_NAME") or "").strip()
    if not name or "${" in name:  # an unfilled catalog default is not a name
        name = f"cashpilot-{device_id[:8]}"
    return name


async def _try_register(slug: str, device_id: str, name: str) -> bool | str:
    """One registration attempt: True, False or REGISTER_PENDING. Never raises."""
    from app.collectors import build_collector

    try:
        collector = build_collector(slug, await database.get_config() or {})
    except Exception:
        return False
    if collector is None:
        return False
    try:
        result = await asyncio.wait_for(collector.register_device(device_id, name), 20)
        return result if result == REGISTER_PENDING else bool(result)
    except Exception as exc:
        logger.warning("Could not add the %s device to the account: %s", slug, notify.redact(str(exc)))
        return False
    finally:
        await collector.close()


async def _register_device(slug: str, device_id: str, env: dict[str, Any]) -> bool | None:
    """Add a deployed device to the user's account with the stored collector credentials.

    Only where the provider's API can do it and the credentials are saved; anything
    else is False, and the deploy form shows the manual step instead. None when
    the provider has not seen the new device yet: the registration is stored and
    retried by the scheduler (_retry_device_links). Never fails the deploy, which
    has already happened.
    """
    result = await _try_register(slug, device_id, _device_name(device_id, env))
    if result == REGISTER_PENDING:
        now = datetime.now(UTC)
        try:
            await database.save_pending_device_link(
                slug, device_id, _device_name(device_id, env), _db_time(now + DEVICE_LINK_FIRST_RETRY)
            )
        except Exception as exc:
            logger.warning("Could not store the %s registration to retry: %s", slug, exc)
            return False
        return None
    return bool(result)


async def _device_still_deployed(slug: str, device_id: str) -> bool:
    """Whether ``device_id`` still runs: a deployment's id, or a proxied copy's derived one."""
    link = (catalog.get_service(slug) or {}).get("device_link") or {}
    if not link.get("id_env"):
        return False
    deployed = await database.get_deployed_env_values(slug, link["id_env"])
    if device_id in deployed:
        return True
    for copy in await database.list_proxied_copies():
        if copy["slug"] != slug:
            continue
        for original in deployed:
            try:
                if proxies.copy_device_id(original, copy["proxy_id"]) == device_id:
                    return True
            except ValueError:
                continue
    return False


async def _retry_device_links() -> None:
    """Retry the registrations a provider asked to try again later, with backoff."""
    now = datetime.now(UTC)
    for row in await database.list_pending_device_links():
        if row["gave_up"] or row["next_at"] > _db_time(now):
            continue
        slug, device_id = row["slug"], row["device_id"]
        if not await _device_still_deployed(slug, device_id):
            await database.delete_pending_device_link(slug, device_id)
            continue
        result = await _try_register(slug, device_id, row["name"])
        if result is True:
            await database.delete_pending_device_link(slug, device_id)
            await database.record_health_event(slug, "device_linked", f"{device_id} added to the account")
            logger.info(
                "Added the %s device %s to the account after %d retries", slug, device_id[:8], row["attempts"] + 1
            )
            continue
        attempts = row["attempts"] + 1
        first = datetime.strptime(row["first_at"], "%Y-%m-%d %H:%M:%S").replace(tzinfo=UTC)
        if now - first >= DEVICE_LINK_GIVE_UP:
            await database.update_pending_device_link(
                slug, device_id, attempts=attempts, next_at=row["next_at"], gave_up=True
            )
            svc = catalog.get_service(slug) or {}
            step = catalog.device_link_step(svc, {(svc.get("device_link") or {}).get("id_env"): device_id}) or {}
            message = (
                f"{svc.get('name') or slug} never accepted the device {device_id} after a day of retries, "
                f"so it earns for nobody. {step.get('steps', '')} {step.get('url', '')}"
            ).strip()
            if alert_id := await database.record_alert("notice", slug, message):
                _spawn(
                    _push_alert(
                        "notice",
                        slug,
                        f"CashPilot: {svc.get('name') or slug} device not on your account",
                        message,
                        alert_id,
                    )
                )
            continue
        wait = min(DEVICE_LINK_FIRST_RETRY * (2**attempts), DEVICE_LINK_MAX_RETRY)
        await database.update_pending_device_link(
            slug, device_id, attempts=attempts, next_at=_db_time(now + wait), gave_up=False
        )


async def _retrying_device_ids() -> dict[str, set[str]]:
    """Per slug, the device ids _retry_device_links is still adding to the account.

    Only inside the retry window: a row the retry job never got to mark as given
    up (the job failing every time) must not hide the notice for ever. A failed
    read answers {}: the plain notice then shows, as it did before.
    """
    try:
        rows = await database.list_pending_device_links()
    except Exception as exc:
        logger.warning("Could not read the device registrations being retried: %s", exc)
        return {}
    window_start = _db_time(datetime.now(UTC) - DEVICE_LINK_GIVE_UP)
    retrying: dict[str, set[str]] = {}
    for row in rows:
        if not row["gave_up"] and str(row.get("first_at") or "") > window_start:
            retrying.setdefault(row["slug"], set()).add(row["device_id"])
    return retrying


def _unlinked_devices_warning(result: Any, retrying: set[str] | frozenset[str] = frozenset()) -> str:
    """The notice for deployed devices the provider says are not on the account.

    A device in ``retrying`` is left out: the app is still adding it, so asking
    the user to add it by hand is wrong, and the notice would take the day's
    push from the give-up message that follows if the retries fail.
    """
    ids = sorted(set(getattr(result, "unlinked_devices", None) or []) - set(retrying))
    if not ids:
        return ""
    svc = catalog.get_service(result.platform) or {}
    link = svc.get("device_link") or {}
    steps = []
    for device_id in ids:
        step = catalog.device_link_step(svc, {link.get("id_env"): device_id}) or {}
        steps.append(f"{device_id} ({step['url']})" if step.get("url") else device_id)
    name = svc.get("name") or result.platform
    which = "A deployed device is" if len(ids) == 1 else f"{len(ids)} deployed devices are"
    earns = "earns" if len(ids) == 1 else "earn"
    return (
        f"{which} not on your {name} account and {earns} for nobody: {'; '.join(steps)}. {link.get('steps', '')}"
    ).strip()


async def _run_collection() -> None:
    """Collect earnings from all deployed services that have collectors."""
    global _collector_alerts, _collected_ok_last_run, _collector_says_disabled
    if _collection_lock.locked():
        logger.info("Collection already in progress, skipping")
        return
    async with _collection_lock:
        success = True
        start_time = 0.0
        collectors: list = []
        try:
            start_time = metrics.record_collection_start()
            deployments = await database.get_deployments()
            config = await database.get_config() or {}
            if not isinstance(config, dict):
                config = {}
            from app.collectors import _close_stale, make_collectors

            collectors = make_collectors(deployments, config)
            await _close_stale()
            for c in collectors:
                try:
                    await _attach_node_readings(c)
                except Exception as exc:  # noqa: BLE001 - the collector reports the empty read
                    logger.warning("Could not read node figures for %s: %s", getattr(c, "platform", "?"), exc)
            # Platforms whose collector was asked to check deployed device ids.
            device_checked: set[str] = set()
            for c in collectors:
                slug = getattr(c, "platform", "")
                link = (catalog.get_service(slug) or {}).get("device_link") if slug else None
                if not isinstance(link, dict):
                    continue
                try:
                    c.device_ids = await database.get_deployed_env_values(slug, link["id_env"])
                except Exception as exc:
                    # Only this service's device check is lost, never the run.
                    logger.warning("Could not read the deployed %s device ids: %s", slug, exc)
                    continue
                if c.device_ids:
                    device_checked.add(slug)
            results = await asyncio.gather(*(_collect_bounded(c) for c in collectors), return_exceptions=True)
            retrying = await _retrying_device_ids()
            alerts: list[dict[str, str]] = []
            # Platforms that were already failing before this run: used to detect a
            # recovery, so a service that breaks again later notifies again rather
            # than being deduped into silence forever.
            previously_alerting = {a["platform"] for a in _collector_alerts}
            # Notices tracked separately: a warning-free success is a notice's
            # recovery, and conflating the two would clear the wrong kind.
            previously_noticing = {a["platform"] for a in _collector_alerts if a.get("kind") == "notice"}
            platforms_ok = 0
            collected_ok: set[str] = set()
            disabled_now: set[str] = set()
            unlinked_ok: set[str] = set()
            for result in results:
                if isinstance(result, Exception):
                    # Redacted BEFORE it is logged. A collector exception is
                    # usually an httpx error that embeds the offending header,
                    # which for several providers IS the live credential.
                    logger.warning("Collector raised exception: %s", notify.redact(str(result)))
                    success = False
                    continue
                if result.error:
                    # Redact FIRST. The comment below explains why the alert is
                    # sanitised; this line used to log the raw string one step
                    # earlier and put the credential in the container log
                    # anyway, defeating the whole exercise.
                    logger.warning("Collection error for %s: %s", result.platform, notify.redact(result.error))
                    # Redact ONCE, here, so the same sanitized string is what gets shown,
                    # stored and sent. Collector errors are usually str(exc), and an httpx
                    # exception embeds the offending header or URL — which for several
                    # providers is a live credential. The alert is now durable and readable
                    # by any authenticated role, so it must never hold a secret.
                    safe_error = notify.redact(result.error)
                    error_kind = getattr(result, "error_kind", None)
                    says_disabled = credential_test.classify(safe_error) == credential_test.ACCOUNT_DISABLED
                    if says_disabled:
                        disabled_now.add(result.platform)
                    entry = {"kind": "collector", "platform": result.platform, "error": safe_error}
                    if error_kind:
                        entry["category"] = error_kind
                    alerts.append(entry)
                    metrics.record_collection_error(result.platform)
                    # Push out-of-band only the FIRST time this failure appears — a
                    # collector broken for a week must not notify every single hour.
                    # A disabled account has its own alert (_account_disabled_check);
                    # a "collector failed" push beside it would point at the
                    # credentials, which are fine.
                    if says_disabled or result.platform in _account_disabled:
                        pass
                    elif alert_id := await database.record_alert(
                        "collector", result.platform, safe_error, category=error_kind
                    ):
                        # The push is the only channel an unattended install
                        # has, so the taxonomy must reach it too: "rotate your
                        # cookie" and "provider hiccup" are different asks.
                        cause = {
                            "auth": " (credential rejected)",
                            "transient": " (provider unreachable)",
                            "shape": " (page changed)",
                        }.get(error_kind or "", "")
                        _spawn(
                            _push_alert(
                                "collector",
                                result.platform,
                                f"CashPilot: {result.platform} collector failed{cause}",
                                safe_error,
                                alert_id,
                            )
                        )
                else:
                    # A caveat on a SUCCESSFUL reading. It is shown, because the
                    # figure is partial and the user should know, but it is not
                    # a failure: the balance below is stored exactly as any other.
                    if unlinked := _unlinked_devices_warning(result, retrying.get(result.platform, set())):
                        result.warning = f"{result.warning} {unlinked}" if result.warning else unlinked
                    if getattr(result, "warning", None):
                        safe_warning = notify.redact(result.warning)
                        logger.info("Collector notice for %s: %s", result.platform, safe_warning)
                        alerts.append({"kind": "notice", "platform": result.platform, "error": safe_warning})
                        # Push out-of-band with the same once-per-window dedupe
                        # errors get. A notice used to reach ONLY the bell,
                        # which is read by whoever happens to open the UI — and
                        # "nobody looked for days" is the incident this class
                        # of warning (an unreachable storj node) comes from.
                        if alert_id := await database.record_alert("notice", result.platform, safe_warning):
                            _spawn(
                                _push_alert(
                                    "notice",
                                    result.platform,
                                    f"CashPilot: {result.platform} needs attention",
                                    safe_warning,
                                    alert_id,
                                )
                            )
                    elif (
                        result.platform in previously_noticing
                        and result.platform in device_checked
                        and getattr(result, "unlinked_devices", None) is None
                    ):
                        # The device check could not finish (provider error, rate
                        # limit): that proves nothing, so the notice stays rather
                        # than being cleared now and pushed again next hour.
                        alerts.extend(
                            a
                            for a in _collector_alerts
                            if a.get("kind") == "notice" and a["platform"] == result.platform
                        )
                    elif result.platform in previously_noticing:
                        # A warning-free success is the notice's recovery: drop
                        # the stored row so the NEXT warning pushes again rather
                        # than being deduped into silence forever.
                        await database.clear_alerts("notice", result.platform)
                    earned = None
                    if getattr(result, "nodes", None) is not None:
                        # Read node by node (issue #488): the balance is every
                        # known node's unsettled amount, and it falls by itself
                        # each time a node settles, so it is never asked about
                        # as a payout. The running total carries the earnings.
                        totals = await database.record_node_figures(result.platform, result.nodes, result.currency)
                        result.balance = totals["balance"]
                        earned = totals["earned"]
                    else:
                        # Compare against the last reading BEFORE writing the new
                        # one: a payout is only visible as the step between two
                        # snapshots.
                        payout_alert = await _detect_payout(result)
                        if payout_alert:
                            alerts.append(payout_alert)
                    await database.upsert_earnings(
                        platform=result.platform,
                        balance=result.balance,
                        currency=result.currency,
                        # Snapshot the rate now. to_usd(1.0, cur) is the cur -> USD
                        # rate (1.0 for USD, None if unknown); rates are only cached
                        # live, so this is the only chance to record what this reading
                        # was actually worth.
                        fx_rate_usd=exchange_rates.to_usd(1.0, result.currency),
                        earned=earned,
                    )
                    logger.info("Collected %s: %.4f %s", result.platform, result.balance, result.currency)
                    platforms_ok += 1
                    collected_ok.add(result.platform)
                    if getattr(result, "unlinked_devices", None):
                        unlinked_ok.add(result.platform)
                    if result.platform in previously_alerting:
                        # Recovered — drop the stored alert so a future failure counts
                        # as new and notifies again.
                        await database.clear_alerts("collector", result.platform)
            # Re-add every payout still awaiting an answer, not just ones
            # detected on THIS run. `record_probable_payout` returns None while
            # one is already pending, so a freshly detected payout produced an
            # alert once and then vanished from the bell on the very next
            # collection — before the user had a chance to confirm or reject
            # it. The pending rows in the database are the real source of
            # truth for "there is still a question outstanding", so the bell is
            # rebuilt from them rather than from what happened to be noticed in
            # the last few seconds.
            alerts.extend(
                await _pending_payout_alerts(seen={a["platform"] for a in alerts if a.get("kind") == "payout"})
            )
            # Flatline is the one failure mode nothing else can surface — the
            # container is up and the collector authenticates fine. It reached
            # the database and the notifier, but never this list, and on a
            # default install no notifier is configured (docker-compose.yml sets
            # none of the NTFY/WEBHOOK/TELEGRAM variables), so the only place the
            # user would ever see it was the one place it never went. The bell
            # said "All collectors healthy".
            try:
                fleet = await _get_all_worker_containers()
            except Exception as exc:  # noqa: BLE001 - unknown fleet: hold the heads-up back
                logger.warning("Could not read the fleet for the flatline check: %s", exc)
                fleet = []
            alerts.extend(await _flatline_check(collected_ok, deployments, fleet, unlinked=unlinked_ok))
            _collected_ok_last_run = collected_ok
            await _record_balance_reads(collected_ok)
            # Sticky until a balance is read: a disabled account's later logins
            # often come back as a rate limit, which says nothing new. A service
            # the user removed has no collector to read that balance, so it
            # leaves here instead, or its bell entry stayed until a restart.
            deployed = {d.get("slug") for d in deployments}
            _collector_says_disabled = ((_collector_says_disabled | disabled_now) - collected_ok) & deployed
            await _account_disabled_check(fleet)
            # Offline workers re-derived every rebuild, or the hourly wholesale
            # replacement of this list would silently un-report a dead machine.
            alerts.extend(await _offline_worker_alerts())
            # Kept by the health check, which runs every five minutes; without
            # this the hourly rebuild would drop a down container from the bell.
            alerts.extend(_container_down_bell.values())
            alerts.extend(_missing_bell.values())
            alerts.extend(_activity_bell)
            alerts.extend(await _proxy_cap_alerts())
            _collector_alerts = _with_account_disabled(alerts)
            if collectors and platforms_ok == 0:
                # A run in which EVERY collector failed used to be recorded as
                # a SUCCESS: _collect_bounded converts exceptions into
                # EarningsResult.error, so the isinstance branch above never
                # fired and nothing else touched the flag. That refreshed
                # collection_last_success_timestamp hourly through a total
                # outage — both documented Prometheus alerts (CollectionFailing,
                # EarningsStale) were structurally unable to fire, proven by
                # execution. A PARTIAL failure stays a success: one chronically
                # broken collector must not freeze the staleness stamp and
                # false-alarm while fourteen others are collecting fine, and an
                # install with no collectors has nothing to be stale about.
                success = False
        except Exception as exc:
            logger.error("Collection run failed: %s", exc)
            success = False
            platforms_ok = 0
            # Keep the per-platform entries: the next run derives "which platforms were
            # failing" from this list, so clobbering it would make every platform
            # permanently unrecoverable — its stored alert would never be cleared and
            # its next real failure would be deduped into silence.
            _collector_alerts = [
                *(a for a in _collector_alerts if a.get("platform") != "collection"),
                {"platform": "collection", "error": "Collection run failed — see server logs"},
            ]
        finally:
            metrics.record_collection_end(start_time, success, platforms_ok, len(collectors))
            # Set even on a failed run: the bell's question is "has anything
            # looked", and a run that tried and failed HAS looked — its failure
            # is in the alert list the bell is about to show.
            global _collection_has_run, _last_collection_finished
            _collection_has_run = True
            # "The machinery ran", success or not — /api/health and the bell
            # measure staleness of THIS stamp. A run that wedges on the
            # collection lock returns early above and never reaches here, so
            # the age keeps growing through exactly the failure the skip line
            # used to disguise as routine.
            _last_collection_finished = time.monotonic()


# ---------------------------------------------------------------------------
# App lifecycle
# ---------------------------------------------------------------------------


async def _run_data_retention() -> None:
    """Purge data older than 400 days."""
    try:
        deleted = await database.purge_old_data()
        if deleted:
            logger.info("Data retention: purged %d old rows", deleted)
    except Exception as exc:
        logger.warning("Data retention error: %s", exc)


#: Config key holding when the last VACUUM finished (ISO 8601, UTC).
_LAST_VACUUM_KEY = "_last_vacuum_at"
_VACUUM_EVERY = timedelta(days=7)


async def _run_vacuum() -> None:
    """Reclaim disk left by retention deletes (SQLite does not auto-shrink).

    Runs at startup and on a weekly timer, but only when the last VACUUM is more
    than a week old. The timer alone counts from process start, and every
    release recreates this container, so an install that restarts more than
    once a week never reached it. The stamp is what makes a startup run cheap:
    a restart soon after a VACUUM does not rewrite the whole file again.
    """
    try:
        last = datetime.fromisoformat(str(await database.get_config(_LAST_VACUUM_KEY)))
        if datetime.now(UTC) - last < _VACUUM_EVERY:
            return
    except Exception:
        pass  # no stamp yet, or an unreadable one: vacuum rather than never
    try:
        await database.vacuum_database()
        await database.set_config(_LAST_VACUUM_KEY, datetime.now(UTC).isoformat())
        logger.info("Database VACUUM complete")
    except Exception as exc:
        logger.warning("Database VACUUM error: %s", exc)


# A recovery re-arms the offline alert only after this many CONSECUTIVE sweeps
# online. One heartbeat is not a recovery: an Android phone in Doze wakes every
# ~12 minutes, heartbeats once and sleeps again — and when that single beat
# cleared the alert row immediately, every nap became a "new" offline episode
# with its own push. Seven identical Telegram messages in ~80 minutes, live.
# Two sweeps (~4 minutes online) is enough to tell a recovery from a blip while
# a genuinely recovered server re-arms almost immediately. In-memory on
# purpose: a UI restart forgets streaks, which at worst re-arms one alert
# early — the 24h record cooldown still caps the damage.
_SUSTAINED_RECOVERY_SWEEPS = 2
#: Sentinel streak value: this recovery's durable clear SUCCEEDED — stop
#: clearing until the next offline episode resets the streak.
_STREAK_CLEARED = -1
_worker_online_streak: dict[str, int] = {}

#: Floor between offline PUSHES for one worker, independent of episodes.
#: Sustained-recovery damping alone was defeated by a phone whose NORMAL duty
#: cycle is 7-10 minutes awake, 2-3 minutes dozing: every wake window passed
#: the sustained bar, re-armed the alert, and the next nap pushed — eight
#: pushes in two hours, live, AFTER the episode damping shipped. A worker
#: whose ordinary operation is rhythmic flapping can never be rate-limited by
#: episode detection, so the cap sits on the notification itself: the first
#: offline push is immediate, repeats for the same worker wait out the floor.
#: The bell and dashboard stay real-time; only the phone-buzz is capped.
#: In-memory on purpose — a restart forgets the floor, which at worst costs
#: one early push.
_WORKER_OFFLINE_PUSH_COOLDOWN_S = 6 * 3600
_worker_last_offline_push: dict[str, float] = {}


async def _check_stale_workers() -> None:
    """Mark workers as offline if stale, and purge never-enrolled workers offline > 1 hour.

    A worker that HAS enrolled a per-worker key (``api_key_enc`` set) is never
    auto-deleted here, even after a long outage: the host persists that same
    key locally and re-presents it on its next heartbeat. Deleting the row
    would leave no match for that key, and since a confirmed worker's shared
    bootstrap key is refused too, it would be rejected forever with no way to
    re-enroll — a permanent fleet lockout after a reboot/maintenance window.
    Only a worker that never completed enrollment is purged automatically;
    removing an enrolled worker is a deliberate action via the UI.

    This sweep is also the ONLY place the offline alert is cleared — see the
    sustained-recovery branch. The heartbeat route deliberately does not clear.
    """
    try:
        workers = await database.list_workers()
    except Exception as exc:
        logger.warning("Stale worker check error: %s", exc)
        return
    now = datetime.now(UTC)
    purge_cutoff = now - timedelta(hours=1)
    global _collector_alerts
    for w in workers:
        try:
            last_hb = w.get("last_heartbeat")
            if not last_hb:
                continue
            cid = w.get("client_id") or w["name"]
            last = datetime.fromisoformat(last_hb).replace(tzinfo=UTC)
            cutoff = now - timedelta(seconds=_stale_after_seconds(w))
            if w["status"] == "online" and last < cutoff:
                _worker_online_streak.pop(cid, None)
                # Conditional on the heartbeat the decision was based on: a
                # recovery heartbeat landing between this sweep's read and its
                # write must win, or the worker would be alerted offline at its
                # exact moment of coming back.
                if not await database.mark_worker_offline_if_unchanged(w["id"], last_hb):
                    continue
                logger.info("Worker '%s' marked offline (last heartbeat: %s)", w["name"], last_hb)
                # The 42-hour incident, UI side: this transition used to be the
                # log line above and nothing else — the UI KNEW and told nobody.
                # Same alert/push/recovery pattern as a failing collector.
                await _raise_worker_offline_alert(w)
            elif w["status"] == "offline":
                _worker_online_streak.pop(cid, None)
                if last < purge_cutoff and not w.get("api_key_enc"):
                    await database.delete_worker(w["id"])
                    logger.info("Purged stale unenrolled worker '%s' (offline since %s)", w["name"], last_hb)
                else:
                    # Still offline: re-attempt the durable alert. record_alert
                    # dedupes inside its window, so this is a no-op two minutes
                    # after a SUCCESSFUL insert — but an insert or push that
                    # FAILED at transition time (DB blip, notifier down) gets
                    # retried instead of being lost to the already-offline
                    # state, which used to make the miss permanent.
                    await _raise_worker_offline_alert(w)
            elif w["status"] == "online":
                # Sustained recovery, and the ONLY place the offline alert is
                # cleared. Clearing re-arms the next push, so a clear on the
                # FIRST heartbeat turned every Doze nap of a phone worker into
                # a fresh alerted episode (~5 pushes/hour, live). The retry
                # state lives in the streak itself: at or past the threshold
                # the clear runs every sweep until it SUCCEEDS (a raise lands
                # in this loop's per-worker handler and the streak stays
                # armed), and only success parks the streak on the sentinel —
                # so a failed durable clear cannot be stranded by the hourly
                # bell rebuild dropping the in-memory entry.
                streak = _worker_online_streak.get(cid, 0)
                if streak == _STREAK_CLEARED:
                    continue
                streak += 1
                _worker_online_streak[cid] = streak
                if streak >= _SUSTAINED_RECOVERY_SWEEPS:
                    await database.clear_alerts("worker", cid)
                    had_bell = any(a.get("kind") == "worker" and a.get("client_id") == cid for a in _collector_alerts)
                    _collector_alerts = [
                        a for a in _collector_alerts if not (a.get("kind") == "worker" and a.get("client_id") == cid)
                    ]
                    _worker_online_streak[cid] = _STREAK_CLEARED
                    if had_bell:
                        logger.info("Cleared offline alert for recovered worker '%s'", w["name"])
        except Exception as exc:
            logger.warning("Stale worker check error for worker '%s': %s", w.get("name", w.get("id")), exc)


#: Windowed health signals that currently hold, per running copy:
#: (slug, worker_id, instance) -> the rendered ``means`` of each one. Rebuilt by
#: every sweep, so a problem that went away clears on the next one. In memory on
#: purpose: a restart forgets it for at most one sweep.
_windowed_hints: dict[tuple[str, int, int], list[str]] = {}
# The startup sweep and the scheduled one must not overlap: the older would
# finish last and put back stale hints.
_windowed_lock = asyncio.Lock()


_DOCKER_POOLS = ipaddress.ip_network("172.16.0.0/12")


def _private_ipv4(url: str) -> str | None:
    """The host of a worker URL when it looks like the box's LAN address, else None.

    A worker on Docker's bridge network auto-detects its CONTAINER address
    (172.17.x and the rest of Docker's 172.16.0.0/12 pools), which no router
    can forward to, so that range is never named.
    """
    host = urllib.parse.urlparse(url or "").hostname or ""
    try:
        addr = ipaddress.ip_address(host)
    except ValueError:
        return None
    if addr.version != 4 or not addr.is_private or addr.is_loopback or addr in _DOCKER_POOLS:
        return None
    return host


async def _render_signal_means(signal: dict[str, Any], slug: str, worker: dict[str, Any] | None) -> str:
    """The signal's ``means`` with ${VAR} filled from what this copy was deployed with.

    ${HOST_ADDRESS} is the box's LAN address when the worker reports one, so the
    hint can name the exact router rule to create.
    """
    service = catalog.get_service(slug) or {}
    env = {var["key"]: str(var.get("default", "")) for var in (service.get("docker") or {}).get("env", [])}
    wid = (worker or {}).get("id")
    recorded = await database.get_deployment_spec(slug, worker_id=wid) if wid is not None else None
    recorded_env = {k: str(v) for k, v in ((recorded or {}).get("env") or {}).items() if str(v)}
    env.update(recorded_env)
    env["HOST_ADDRESS"] = _private_ipv4((worker or {}).get("url", "")) or "this machine's LAN address"
    text = " ".join(str(signal.get("means") or "").split())
    # A copy deployed before a setting existed does not run with its default yet
    # (Mysterium without UDP_PORTS picks from 10000:60000), so naming the default
    # alone would send the operator to forward a range the node does not use.
    used = set(re.findall(r"\$\{(\w+)\}", text)) - {"HOST_ADDRESS"}
    if recorded is not None and any(key not in recorded_env for key in used):
        text = (
            "This copy was deployed before CashPilot set these values, so it does not use them yet: "
            "redeploy it from the dashboard first. " + text
        )
    return re.sub(r"\$\{(\w+)\}", lambda m: env.get(m.group(1), m.group(0)), text)


async def _check_windowed_signals() -> None:
    """Judge every windowed health signal of every running copy (producer_state.is_windowed).

    A copy whose read fails keeps its last verdict rather than losing it to a
    blip; a copy no longer running, or no longer declaring the signal, drops it.
    """
    async with _windowed_lock:
        await _sweep_windowed_signals()


async def _sweep_windowed_signals() -> None:
    global _windowed_hints
    try:
        containers = await _get_all_worker_containers()
    except Exception as exc:
        logger.warning("Windowed signal check skipped: %s", exc)
        return
    hints: dict[tuple[str, int, int], list[str]] = {}
    workers: dict[int, dict[str, Any] | None] = {}
    for c in containers:
        slug = egress.container_slug(c)
        wid = c.get("_worker_id")
        if not slug or wid is None or str(c.get("status", "")).lower() != "running":
            continue
        signals = [s for s in producer_state.signals_for(catalog.get_service(slug)) if producer_state.is_windowed(s)]
        if not signals:
            continue
        instance = int(c.get("instance") or 1)
        key = (slug, wid, instance)
        found: list[str] = []
        try:
            if wid not in workers:
                workers[wid] = await database.get_worker(wid)
            for signal in signals:
                params: dict[str, Any] = {
                    "lines": 1000,
                    "match": producer_state.window_filter(signal),
                    "since_hours": float(signal["window_hours"]),
                }
                if instance > 1:
                    params["instance"] = instance
                # Hours of a chatty node's logs: seconds on a fast box, more on a Pi.
                resp = await _proxy_to_worker(wid, "GET", f"/api/containers/{slug}/logs", params=params, timeout=120)
                # A worker older than the filter returns the plain tail, which is
                # not the window: no verdict from it.
                if not resp.get("filtered"):
                    continue
                if producer_state.windowed_hit(resp.get("logs", ""), signal):
                    found.append(await _render_signal_means(signal, slug, workers[wid]))
        except Exception as exc:
            logger.warning("Windowed signal check failed for %s on worker %s: %s", slug, wid, exc)
            if key in _windowed_hints:
                hints[key] = _windowed_hints[key]
            continue
        if found:
            hints[key] = found
    _windowed_hints = hints


async def _raise_worker_offline_alert(w: dict[str, Any]) -> None:
    """Record + push + bell an offline worker, keyed by client_id.

    client_id is the identity, per this codebase's own rule: workers.name is
    the container hostname, cosmetic, mutable across recreates, and shareable
    by two hosts — keying alerts on it would let one worker suppress or clear
    another's. The display name travels in the message and title only.
    """
    cid = w.get("client_id") or w["name"]
    msg = (
        f"Worker '{w['name']}': no heartbeat for over {_stale_after_seconds(w) // 60} minutes. Its "
        "containers keep running and earning, but CashPilot cannot see or manage them until it reconnects."
    )
    if alert_id := await database.record_alert("worker", cid, msg):
        # The push floor, checked only on a FRESH row (a deduped record never
        # gets here): a new episode inside the floor keeps its row and bell
        # entry — the dashboard tells the truth — but does not buzz the phone
        # again. Deliberately never reset on recovery; resetting there is
        # exactly the re-arm loop this exists to end.
        now = time.monotonic()
        last_push = _worker_last_offline_push.get(cid)
        if last_push is None or now - last_push >= _WORKER_OFFLINE_PUSH_COOLDOWN_S:
            _worker_last_offline_push[cid] = now
            _spawn(
                _push_alert(
                    "worker",
                    cid,
                    f"CashPilot: worker '{w['name']}' went offline",
                    msg,
                    alert_id,
                )
            )
        else:
            logger.info(
                "Offline push for worker '%s' suppressed by the per-worker floor (%.0f min since last)",
                w["name"],
                (now - last_push) / 60,
            )
    # Into the bell NOW (the sweep runs every 2 min); the hourly collection
    # rebuild re-derives it from the workers table.
    if not any(a.get("kind") == "worker" and a.get("client_id") == cid for a in _collector_alerts):
        _collector_alerts.append({"kind": "worker", "platform": w["name"], "client_id": cid, "error": msg})


FLEET_API_KEY = fleet_key.resolve_fleet_key()
HOSTNAME_PREFIX = os.getenv("CASHPILOT_HOSTNAME_PREFIX", "cashpilot")
COLLECT_INTERVAL_MIN = int(os.getenv("CASHPILOT_COLLECT_INTERVAL", "60"))
STALE_WORKER_SECONDS = 180  # Mark worker offline after 3 missed heartbeats
# A phone reports from Android's WorkManager, whose shortest period is 15
# minutes, so 180 s would show every phone offline between two reports.
# Three missed periods, the same rule as the 60 s Docker workers.
STALE_ANDROID_WORKER_SECONDS = 45 * 60


def _stale_after_seconds(w: dict[str, Any]) -> int:
    """How long this worker may stay silent before it is marked offline."""
    info = w.get("system_info") or {}
    if not isinstance(info, dict):  # raw DB rows carry it as JSON text
        info = _safe_json(info, {})
    if placement.is_android(info):
        return STALE_ANDROID_WORKER_SECONDS
    return STALE_WORKER_SECONDS


async def _warm_session_epochs() -> None:
    """Restore the in-memory per-user session-epoch cache from durable state.

    Two sources are merged, taking the later timestamp per user:
      - ``users.password_changed_at`` — a password change invalidates older sessions.
      - ``session_revocations.revoked_before`` — a delete/demote invalidates older
        sessions and survives the users row being deleted.

    This runs at startup so those invalidations survive a UI restart. Without the
    revocation half, a delete/demote (which only bumps the in-memory epoch) would be
    forgotten on the next restart and the account's still-valid cookie would be
    honored again with its old role — the bug this fixes.
    """
    epochs: dict[int, float] = {}
    for _u in await database.list_users_with_pwd_epoch():
        changed = _u.get("password_changed_at") or 0.0
        if changed:
            epochs[_u["id"]] = changed
    for _r in await database.list_session_revocations():
        uid = _r["user_id"]
        epochs[uid] = max(epochs.get(uid, 0.0), _r["revoked_before"] or 0.0)
    for uid, ts in epochs.items():
        auth.set_user_pwd_epoch(uid, ts)


@asynccontextmanager
async def lifespan(app: FastAPI):
    # Startup
    # Before anything touches credentials: refuse to run when the encryption key
    # cannot survive a restart. Continuing would encrypt every credential entered
    # during this run under a key that dies with the process.
    database.verify_encryption_key_persisted()
    await database.init_db()
    await database.connect_shared()
    # Warm the per-user session-epoch cache (password changes + delete/demote
    # revocations) so invalidated sessions are rejected without a DB hit in the
    # request path — and, crucially, so those invalidations survive a restart.
    await _warm_session_epochs()
    # Restore the collector alerts persisted by earlier runs, so a restart doesn't
    # silently clear the notification bell while the underlying collector is still
    # broken (previously these lived only in memory).
    await _warm_collector_alerts()
    await _refresh_activity_bell()
    # Credentials saved before saving them began creating a tracking row are
    # otherwise inert forever, while Settings shows them as "Configured".
    await _track_fully_configured_services()
    # First-run setup token: while no users exist, require a one-time token
    # (printed below) for /register so a proxy-exposed instance cannot be seized
    # by the first public visitor. Persisted in config so it survives restarts;
    # cleared once the owner account is created.
    if not await database.has_any_users():
        _tok = await database.get_config("_setup_token")
        if not _tok:
            _tok = setup_token.generate()
            await database.set_config("_setup_token", _tok)
        setup_token.set_active(_tok)
        logger.warning(
            "FIRST-RUN SETUP: no account exists yet. Open /register and enter this "
            "one-time setup token to create the owner account: %s  (shown only here; "
            "not embedded in any URL so it stays out of proxy logs and browser history)",
            _tok,
        )
    if not notify.is_enabled():
        # The push path exists so problems reach someone when nobody is
        # watching the dashboard — and a default install has never armed it.
        # Say so once, loudly, in the same place the setup token announces
        # itself; anything quieter and this line is its own counterexample.
        logger.warning(
            "No notification channel configured — alerts will only appear in the UI bell, "
            "which somebody has to open. Set CASHPILOT_NTFY_URL, CASHPILOT_WEBHOOK_URL or "
            "the CASHPILOT_TELEGRAM_* variables to get collector failures, offline workers "
            "and payout prompts pushed out-of-band."
        )

    catalog.load_services()
    catalog.register_sighup()

    def _on_job_event(event):
        logger.error("Scheduler job %s failed or missed", event.job_id, exc_info=getattr(event, "exception", None))

    scheduler.add_listener(_on_job_event, EVENT_JOB_ERROR | EVENT_JOB_MISSED)
    scheduler.add_job(
        _run_collection,
        "interval",
        minutes=COLLECT_INTERVAL_MIN,
        id="collect",
        max_instances=1,
        coalesce=True,
        misfire_grace_time=300,
    )
    scheduler.add_job(
        _retry_device_links,
        "interval",
        minutes=3,
        id="retry_device_links",
        max_instances=1,
        coalesce=True,
        misfire_grace_time=300,
    )
    scheduler.add_job(
        _run_health_check,
        "interval",
        minutes=5,
        id="health_check",
        max_instances=1,
        coalesce=True,
        misfire_grace_time=300,
    )
    scheduler.add_job(
        _check_stale_workers,
        "interval",
        minutes=2,
        id="stale_workers",
        max_instances=1,
        coalesce=True,
        misfire_grace_time=300,
    )
    scheduler.add_job(
        _check_windowed_signals,
        "interval",
        minutes=30,
        id="windowed_signals",
        max_instances=1,
        coalesce=True,
        misfire_grace_time=300,
    )
    scheduler.add_job(
        _run_data_retention,
        "interval",
        hours=24,
        id="data_retention",
        max_instances=1,
        coalesce=True,
        misfire_grace_time=300,
    )
    # Activity (docs/activity.md): the raw-address purge, the spike check and
    # the network table refresh. Hourly, so no address outlives a day by more
    # than an hour even when no heartbeat arrives to purge on ingest.
    scheduler.add_job(
        _run_activity_hourly,
        "interval",
        hours=1,
        id="activity_hourly",
        max_instances=1,
        coalesce=True,
        misfire_grace_time=300,
    )
    scheduler.add_job(
        _run_vacuum,
        "interval",
        weeks=1,
        id="db_vacuum",
        max_instances=1,
        coalesce=True,
        misfire_grace_time=300,
    )
    # Frequent tick, daily fetch: refresh() asks GitHub once a day after a check
    # that worked and an hour after one that failed, so a rate-limited startup
    # check no longer leaves the release unknown for a whole day. A tick that
    # is not due fetches nothing. Every failure mode inside refresh() lands on
    # UNKNOWN, so a firewalled install produces no noise at all.
    scheduler.add_job(
        update_check.refresh,
        "interval",
        seconds=update_check.SCHEDULER_TICK_SECONDS,
        id="update_check",
        max_instances=1,
        coalesce=True,
        # 300 like every other job here rather than a carve-out. A missed daily
        # check simply happens tomorrow; that is not worth an exception to a
        # convention a test enforces uniformly.
        misfire_grace_time=300,
    )
    # The Android client's own release track, on the same cadence and with
    # the same failure contract. Separate job rather than folded into the one
    # above so a GitHub hiccup fetching one cannot suppress the other.
    scheduler.add_job(
        update_check.refresh_android,
        "interval",
        seconds=update_check.SCHEDULER_TICK_SECONDS,
        id="update_check_android",
        max_instances=1,
        coalesce=True,
        misfire_grace_time=300,
    )
    # Proxy traffic caps that reset on a day of the month (app/proxies.py).
    scheduler.add_job(
        _run_proxy_cap_resets,
        "interval",
        hours=1,
        id="proxy_cap_resets",
        max_instances=1,
        coalesce=True,
        misfire_grace_time=300,
    )
    scheduler.add_job(
        exchange_rates.refresh,
        "interval",
        minutes=15,
        id="exchange_rates",
        max_instances=1,
        coalesce=True,
        misfire_grace_time=300,
    )
    scheduler.start()
    await exchange_rates.refresh()
    # Kick both release checks NOW rather than waiting out the first 24-hour
    # interval. APScheduler's interval trigger does not fire on start, and this
    # container is recreated on every deploy -- so on an install that ships more
    # often than daily the check would never run at all, and the update banner
    # would sit permanently at known=false. Spawned, not awaited: a slow or
    # unreachable GitHub must not delay startup, and every failure inside these
    # already lands on UNKNOWN.
    _spawn(update_check.refresh())
    _spawn(update_check.refresh_android())
    # Retention too, for the same reason: its 24-hour interval counts from
    # process start, so an install that restarts more than once a day (every
    # release recreates this container) never purged anything at all.
    _spawn(_run_data_retention())
    # And VACUUM, which has the same trap on a weekly interval; it checks its
    # own last run, so this rewrites the file at most once a week.
    _spawn(_run_vacuum())
    # The Activity purge too: its hourly timer restarts with the process, so a
    # UI restarted more often than hourly would keep addresses past a day.
    _spawn(_run_activity_purge())
    # And the proxy cap reset: its hourly timer restarts with the process too, so
    # a UI restarted more often than hourly would leave a capped proxy's copies
    # stopped past its reset day.
    _spawn(_run_proxy_cap_resets())
    _spawn(_run_collection())
    _spawn(_check_windowed_signals())
    logger.info("CashPilot UI started (container ops via workers)")

    yield

    # Shutdown
    scheduler.shutdown(wait=False)
    await database.close_shared()
    from app.collectors import close_all_collectors

    await close_all_collectors()
    logger.info("CashPilot stopped")


app = FastAPI(
    title="CashPilot",
    version=version.current(),
    lifespan=lifespan,
    # Off by default. FastAPI serves these unauthenticated, and this app's
    # schema is a map of its own admin surface — every route, parameter and
    # body shape, including the worker and payout endpoints. Nothing here
    # needs them in a self-hosted deployment.
    docs_url=None,
    redoc_url=None,
    openapi_url=None,
)
metrics.setup(app)


# Whether a trusted reverse proxy sits in front (opt-in). Only then do we believe an
# X-Forwarded-Proto header for deciding HTTPS (matches app.deps._TRUST_PROXY semantics;
# read here directly to avoid importing deps into the middleware).
_HSTS_TRUST_PROXY = os.getenv("CASHPILOT_TRUSTED_PROXY", "").strip().lower() in ("1", "true", "yes", "on")


# Security headers middleware
class _SecurityHeadersMiddleware(BaseHTTPMiddleware):
    async def dispatch(self, request, call_next):
        # A fresh nonce per response. Templates read it via request.state so the
        # value in the header and the value in the markup can never disagree —
        # if they did, every inline script would be blocked and the UI would
        # simply stop working.
        nonce = secrets.token_urlsafe(16)
        request.state.csp_nonce = nonce
        response = await call_next(request)
        response.headers["X-Content-Type-Options"] = "nosniff"
        response.headers["X-Frame-Options"] = "DENY"
        response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
        response.headers["Permissions-Policy"] = "camera=(), microphone=(), geolocation=()"
        # script-src no longer allows 'unsafe-inline' (bead guw). Inline event
        # handlers are gone — every control goes through one delegated listener —
        # and the few remaining inline <script> blocks carry a per-response
        # nonce. That matters because this UI renders provider-supplied strings:
        # with 'unsafe-inline', any markup an attacker gets into one of those
        # executes.
        #
        # style-src keeps 'unsafe-inline' for now: the templates carry inline
        # style= attributes, which a nonce cannot cover, and a style injection
        # cannot execute script. Narrower than script, and honest about staying.
        #
        # base-uri/object-src/form-action are additive hardening that costs
        # nothing: block <base> hijack of relative script URLs, disallow plugins,
        # and stop a form POSTing credentials off-origin if an XSS is ever found.
        response.headers["Content-Security-Policy"] = (
            "default-src 'self'; "
            f"script-src 'self' https://cdn.jsdelivr.net 'nonce-{nonce}'; "
            "style-src 'self' https://fonts.googleapis.com 'unsafe-inline'; "
            "font-src 'self' https://fonts.gstatic.com; "
            "img-src 'self' data:; "
            "connect-src 'self' https://cdn.jsdelivr.net; "
            "frame-ancestors 'none'; "
            "base-uri 'none'; "
            "object-src 'none'; "
            "form-action 'self'"
        )
        # HSTS only when the request is actually HTTPS, so plain-HTTP local dev is never
        # pinned to https (which would hard-break it). Behind a trusted proxy that
        # terminates TLS the app sees http, so honor X-Forwarded-Proto there.
        proto = request.headers.get("x-forwarded-proto") if _HSTS_TRUST_PROXY else request.url.scheme
        if proto == "https":
            response.headers["Strict-Transport-Security"] = "max-age=63072000; includeSubDomains"
        return response


app.add_middleware(_SecurityHeadersMiddleware)


@app.exception_handler(RequestValidationError)
async def _validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
    """FastAPI's 422 body echoes the offending input, and some inputs cannot be
    encoded — so the rejection itself became a 500.

    JSON has no ``NaN`` or ``Infinity``. Python's parser accepts them anyway, so
    a client can put one in a float field; pydantic correctly rejects it, and
    then the default handler tries to serialise ``{"input": nan}`` and Starlette's
    encoder raises "Out of range float values are not JSON compliant". The client
    gets an opaque 500 for what is squarely a bad request, and the log fills with
    a traceback that names the encoder rather than the cause.

    Non-finite floats are therefore rendered as their names. That keeps the
    message diagnostic ("we saw NaN") instead of dropping the field, and it fixes
    the whole class rather than the one endpoint that happens to take a float
    today. Every other error is passed through byte-for-byte, so the response
    shape callers already parse is unchanged.
    """
    # jsonable_encoder FIRST, exactly as FastAPI's own handler does: a custom
    # validator's error carries the raised ValueError OBJECT in ctx, which is not
    # serialisable either. Encoding then sanitising fixes both without changing
    # the body for any error that was already fine.
    errors = exc.errors()
    if request.url.path.startswith("/api/proxies"):
        # A proxy body carries its password or auth key, and the default 422
        # echoes the rejected input.
        errors = [{k: v for k, v in err.items() if k not in ("input", "ctx")} for err in errors]
    return JSONResponse(status_code=422, content={"detail": _json_safe(jsonable_encoder(errors))})


def _json_safe(value: Any) -> Any:
    """Recursively replace values the JSON encoder refuses — currently only
    non-finite floats.

    Deliberately narrow. ``bool`` needs no special case: it subclasses ``int``,
    not ``float``, so the check below never sees it. An earlier version guarded
    for it anyway, with a comment that was simply wrong; a negative control
    showed removing the guard changed nothing, so it went.
    (``test_booleans_survive_as_booleans`` still pins the guarantee.)
    """
    if isinstance(value, dict):
        return {k: _json_safe(v) for k, v in value.items()}
    if isinstance(value, (list, tuple)):
        return [_json_safe(v) for v in value]
    if isinstance(value, float) and not math.isfinite(value):
        return repr(value)  # "nan", "inf", "-inf"
    return value


# Static files
app.mount("/static", StaticFiles(directory="app/static"), name="static")


# ---------------------------------------------------------------------------
# Auth helpers + templates (shared, defined in app.deps).
#
# Imported here so ``app.main._require_owner`` / ``app.main.templates`` keep
# resolving for tests and for the split router groups, which reference them
# through the ``app.main`` namespace (e.g. ``main._require_owner``).
# ---------------------------------------------------------------------------
from app.deps import (  # noqa: E402
    _login_redirect,  # noqa: F401  (re-exported for app.main.* test/router surface)
    _require_auth_api,
    _require_first_run_access,  # noqa: F401  (re-exported for app.main.* router surface)
    _require_owner,
    _require_private_network,  # noqa: F401  (re-exported for app.main.* router surface)
    _require_reader,
    _require_writer,
    client_ip,  # noqa: F401  (re-exported for app.main.* router surface)
    templates,  # noqa: F401  (re-exported for app.main.* router/test surface)
)

# ---------------------------------------------------------------------------
# API: Services
# ---------------------------------------------------------------------------


@app.get("/api/mode")
async def api_mode(request: Request) -> dict[str, Any]:
    """Return CashPilot operating mode and Docker availability."""
    _require_auth_api(request)
    return {"docker": False, "mode": "ui"}


@app.get("/api/services")
async def api_list_services(request: Request) -> list[dict[str, Any]]:
    _require_auth_api(request)
    return catalog.get_services()


def _collector_needs_setup(slug: str, config: dict[str, str]) -> bool:
    """True if `slug` has an earnings collector whose required config is unset.

    A service can be deployed and earning while CashPilot still can't read its
    balance because the (separate) collector credentials haven't been entered.
    This distinguishes that "not set up yet" state from a real collector error.
    """
    from app.collectors import _COLLECTOR_ARGS, COLLECTOR_MAP

    if slug not in COLLECTOR_MAP:
        return False
    for arg in _COLLECTOR_ARGS.get(slug, []):
        if arg.startswith("?"):  # optional arg — not required for setup
            continue
        if not config.get(f"{slug}_{arg}", ""):
            return True
    return False


# Catalog statuses a service must never be deployed with. Shared by the catalog
# listing (which hides them) and the deploy gate (which refuses them) so the two
# can't drift apart again — previously the listing hid all three but deploy only
# refused "dead", so a direct link or a stale page could still deploy a broken or
# dropped service and then silently earn nothing.
from app.constants import MAX_INSTANCES  # noqa: E402
from app.constants import UNDEPLOYABLE_STATUSES as _UNDEPLOYABLE_STATUSES  # noqa: E402 - single source, see constants

# One definition each, shared with custom services (app/custom_services.py).
_split_image = custom_services.split_image


# The architecture table and the per-arch image resolver live in app/arch.py so
# the preflight, the compose export and the deploy proxy reason in one vocabulary.
_ARCH_FAMILY = arch.MACHINE_FAMILY
_image_for_arch = arch.image_for


def _image_outdated(deployed: str, catalog_image: str) -> bool:
    """True when a running container's image no longer matches the catalog entry.

    Flags the case a provider changed its image path (the ProxyBase migration) or the
    catalog re-pinned to a new digest — so the dashboard can prompt a re-deploy instead
    of showing a healthy-looking container that is silently running a retired image.
    Deliberately conservative: unknown/empty images and a pure tag-vs-digest difference
    of the same repository are NOT flagged.
    """
    if not deployed or not catalog_image:
        return False
    # A bare image ID: the image has neither a tag nor a digest (loaded or built
    # locally), so there is no repository to compare. Unknown, not outdated.
    if deployed.startswith("sha256:"):
        return False
    d_repo, _, d_digest = _split_image(deployed)
    c_repo, _, c_digest = _split_image(catalog_image)
    if d_repo != c_repo:
        return True
    return bool(c_digest and d_digest and c_digest != d_digest)


def _docker_hub_short(ref: str) -> str:
    """docker.io/library/busybox:1 as Docker reports it back: busybox:1."""
    for prefix in (
        "docker.io/library/",
        "index.docker.io/library/",
        "registry-1.docker.io/library/",
        "docker.io/",
        "index.docker.io/",
        "registry-1.docker.io/",
        "library/",
    ):
        if ref.startswith(prefix):
            return ref[len(prefix) :]
    return ref


def _custom_image_outdated(deployed: str, wanted: str) -> bool:
    """Whether a box runs another image than the custom service's definition.

    Stricter than _image_outdated, which ignores a tag change of the same
    repository on purpose: a custom service's reference is the user's own
    choice, so any edit (a tag, a digest, a switch between the two) shows the
    redeploy prompt. Docker Hub spellings are compared in their short form, and
    no tag reads as "latest", which is what Docker pulls and reports. A bare
    image ID tells nothing, so it is never flagged.
    """
    deployed, wanted = _docker_hub_short(deployed), _docker_hub_short(wanted)
    if not deployed or not wanted or deployed.startswith("sha256:"):
        return False
    d_repo, d_tag, d_digest = _split_image(deployed)
    w_repo, w_tag, w_digest = _split_image(wanted)
    if d_repo != w_repo or bool(d_digest) != bool(w_digest):
        return True
    if w_digest:
        return d_digest != w_digest
    return (d_tag or "latest") != (w_tag or "latest")


def _apply_service_meta(entry: dict[str, Any], svc: dict[str, Any] | None) -> None:
    """Attach the catalog-derived cashout / referral / website fields to a deployed
    entry. Shared by the container-backed and external paths; a no-op when the catalog
    no longer lists the deployed slug."""
    if not svc:
        return
    cashout = svc.get("cashout", {})
    if cashout:
        entry["cashout"] = cashout
    referral = svc.get("referral", {})
    if referral:
        entry["referral_url"] = referral.get("signup_url", "")
    entry["website"] = svc.get("website", "")
    entry["support_url"] = svc.get("support_url", "")


#: Config key holding the muted drift warnings: {"<slug>|<worker id>": catalog fingerprint}.
_DRIFT_IGNORED_KEY = "drift_ignored"
# The map is read, extended and written back whole; two Ignores at once must not drop one.
_drift_ignore_lock = asyncio.Lock()


def _drift_ignored(config: dict[str, Any]) -> dict[str, str]:
    raw = _safe_json(config.get(_DRIFT_IGNORED_KEY) or "{}", {})
    return {str(k): str(v) for k, v in raw.items()} if isinstance(raw, dict) else {}


@app.post("/api/services/{slug}/drift/ignore")
async def api_drift_ignore(
    request: Request, slug: str, worker_id: int, _auth: dict[str, Any] = Depends(_require_owner)
) -> dict[str, str]:
    """Mute the drift warning of ``slug`` on one worker until the catalog changes it.

    For a container the owner changed on purpose. The mute is tied to the
    catalog's fingerprint of the judged fields, so a later catalog fix to this
    service shows again instead of hiding behind an old decision.
    """
    svc = catalog.get_service(slug)
    if not svc:
        raise HTTPException(status_code=404, detail=f"Service '{slug}' not found")
    async with _drift_ignore_lock:
        cfg = await database.get_config()
        ignored = _drift_ignored(cfg if isinstance(cfg, dict) else {})
        ignored[f"{slug}|{worker_id}"] = drift.catalog_fingerprint(svc)
        await database.set_config(_DRIFT_IGNORED_KEY, json.dumps(ignored, sort_keys=True))
    return {"status": "ignored"}


async def _instance_drift(
    svc: dict[str, Any] | None,
    slug: str,
    inst: dict[str, Any],
    custom_image: str | None,
    specs: dict[tuple[str, int], dict[str, Any] | None],
) -> tuple[list[str], bool]:
    """How this running copy differs from what a redeploy would build, and whether it has a record.

    Never raises: a record that cannot be read leaves the copy judged on the
    catalog alone, and a check that fails says nothing rather than blanking
    the dashboard.
    """
    wid = inst.get("_worker_id")
    try:
        if wid is not None and (slug, wid) not in specs:
            specs[(slug, wid)] = await database.get_deployment_spec(slug, worker_id=wid)
    except Exception as exc:
        logger.warning("Could not read the deployment record of %s on worker %s: %s", slug, wid, exc)
        specs[(slug, wid)] = None
    recorded = specs.get((slug, wid)) if wid is not None else None
    try:
        found = drift.container_drift(
            svc, inst.get("shape"), recorded, custom_image=custom_image, worker_arch=inst.get("_arch")
        )
    except Exception as exc:
        logger.warning("Drift check failed for %s on worker %s: %s", slug, wid, exc)
        return [], False
    return found, recorded is not None


@app.get("/api/services/deployed")
async def api_services_deployed(request: Request) -> list[dict[str, Any]]:
    """Return deployed services with container status, balance, CPU, memory.

    Multiple containers for the same slug (multi-node) are aggregated into a
    single row with summed CPU/memory, an instance count, and per-instance
    details for the expandable sub-row UI.
    """
    _require_reader(request)
    statuses: list[dict[str, Any]] = await _get_all_worker_containers()

    # Get latest earnings per platform for balance display
    earnings = await database.get_earnings_summary()
    balance_map = {e["platform"]: e["balance"] for e in earnings}
    currency_map = {e["platform"]: e["currency"] for e in earnings}

    # Get health scores
    health_scores = await database.get_health_scores(7)
    health_map = {h["slug"]: h for h in health_scores}

    # Slugs whose balance could not be read, shown as "can't read balance".
    # Only a collector alert means that. Every other kind rides on a reading
    # that SUCCEEDED or is about something else: a payout question exists
    # because the balance was read and dropped, a flatline because it was read
    # and did not move, a notice is shown on the row as itself, a down
    # container has its own alert, and a worker alert names a worker, not a
    # service. Listing the exceptions let each new kind leak in as a failure.
    alert_slugs = {a["platform"] for a in _collector_alerts if a.get("kind", "collector") == "collector"}
    notice_map = {a["platform"]: a.get("error", "") for a in _collector_alerts if a.get("kind") == "notice"}

    # Config (decrypted) to detect collectors whose credentials aren't set yet.
    # A config-read failure must not blank the dashboard — degrade to "unknown".
    config: dict[str, str] = {}
    try:
        cfg = await database.get_config()
        if isinstance(cfg, dict):
            config = cfg
    except Exception as exc:
        logger.warning("Could not load config for collector-setup check: %s", exc)

    # Aggregate by slug: one row per service
    # "unknown" ranks LAST on purpose: best_status picks the lowest number, so a
    # service that is running on one worker and unknown on another reports
    # running -- a known fact beats a blind spot. It only wins when nothing else
    # is known, which is the honest answer at that point.
    # "stopped" is what _android_app_status emits; Docker's own word is "exited".
    # It MUST be listed: an unlisted status falls through to .get(cur, 9), and
    # with unknown ranked 5 a blind spot would have beaten a definite stopped --
    # the exact inverse of the rule below. (CodeRabbit, PR #252.)
    #
    # "unknown" ranks LAST on purpose: best_status picks the lowest number, so a
    # service running on one worker and unknown on another reports running -- a
    # known fact beats a blind spot. It only wins when nothing else is known.
    _STATUS_PRIORITY = {
        "running": 0,
        "restarting": 1,
        "exited": 2,
        "stopped": 2,
        "created": 3,
        "dead": 4,
        "unknown": 5,
    }
    slug_agg: dict[str, dict[str, Any]] = {}
    for s in statuses:
        slug = s["slug"]
        if slug not in slug_agg:
            slug_agg[slug] = {
                "instances": [],
                "total_cpu": 0.0,
                "total_mem": 0.0,
                "best_status": s.get("status", "unknown"),
                "image": s.get("image", ""),
            }
        agg = slug_agg[slug]
        agg["instances"].append(s)
        # Unknown instances are counted separately rather than summed as zero:
        # averaging a failed stats read into the total drags the figure toward
        # zero and makes a busy service look idle. `.get(key, 0)` does not help
        # here — the key EXISTS with value None, so the default never applies.
        cpu, mem = s.get("cpu_percent"), s.get("memory_mb")
        if cpu is None or mem is None:
            agg["unknown_stats"] = agg.get("unknown_stats", 0) + 1
        else:
            agg["total_cpu"] += float(cpu)
            agg["total_mem"] += float(mem)
            agg["measured"] = agg.get("measured", 0) + 1
        cur = s.get("status", "unknown")
        if _STATUS_PRIORITY.get(cur, 9) < _STATUS_PRIORITY.get(agg["best_status"], 9):
            agg["best_status"] = cur

    result = []
    # Custom definitions, read only when a custom container is on the fleet, so
    # a dashboard without one asks the database nothing new.
    customs: dict[str, dict[str, Any]] = {}
    if any(custom_services.is_custom_slug(slug) for slug in slug_agg):
        try:
            customs = {row["slug"]: row["definition"] for row in await database.list_custom_services()}
        except Exception as exc:
            logger.warning("Could not load custom services for the dashboard: %s", exc)
    # Deployment records read for the drift check, one per (slug, worker).
    specs: dict[tuple[str, int], dict[str, Any] | None] = {}
    drift_on = constants.drift_warnings_enabled()
    drift_ignored = _drift_ignored(config) if drift_on else {}
    for slug, agg in slug_agg.items():
        svc = catalog.get_service(slug)
        health = health_map.get(slug, {})
        custom_images = await database.get_custom_images(slug) if svc else {}

        # Build per-instance detail list (local first)
        instance_details = []
        for inst in agg["instances"]:
            detail = {
                "node": inst.get("_node", "unknown"),
                "worker_id": inst.get("_worker_id"),
                "status": inst.get("status", "unknown"),
                # None reaches the UI as null, which renders as an em dash. A
                # formatted "0.00" would be indistinguishable from a real idle
                # container.
                "cpu": None if inst.get("cpu_percent") is None else f"{float(inst['cpu_percent']):.2f}",
                "memory": None if inst.get("memory_mb") is None else f"{float(inst['memory_mb']):.1f} MB",
                "container_name": inst.get("name", ""),
                "has_docker": inst.get("_has_docker", False),
                "is_android": inst.get("_is_android", False),
                # Started outside CashPilot: it has no CashPilot label, so the
                # container commands cannot target it and the UI must not offer
                # them.
                "unmanaged": inst.get("deployed_by") == "external",
                # Which copy on that worker; the UI labels copies 2+ as "#N".
                "instance": inst.get("instance") or 1,
                "custom_image": custom_images.get(inst.get("_worker_id")),
                # What a windowed health signal says is wrong with this copy.
                "hints": _windowed_hints.get((slug, inst.get("_worker_id"), int(inst.get("instance") or 1)), []),
                "fatal": inst.get("fatal"),
            }
            # Only containers CashPilot made (constants.CASHPILOT_DEPLOYED): a
            # container the user runs by hand is theirs to shape, even when it
            # carries CashPilot's labels or a catalog image.
            if (
                drift_on
                and isinstance(inst.get("shape"), dict)
                and inst.get("deployed_by") in constants.CASHPILOT_DEPLOYED
                and drift_ignored.get(f"{slug}|{inst.get('_worker_id')}") != drift.catalog_fingerprint(svc)
            ):
                drift_found, has_record = await _instance_drift(
                    svc, slug, inst, custom_images.get(inst.get("_worker_id")), specs
                )
                if drift_found:
                    detail["drift"] = drift_found
                    detail["drift_has_record"] = has_record
            if inst.get("_is_android"):
                detail["net_tx_24h"] = inst.get("_net_tx_24h", 0)
                detail["net_rx_24h"] = inst.get("_net_rx_24h", 0)
            instance_details.append(detail)
        # Sort: local first, then alphabetically by node name
        instance_details.sort(key=lambda x: (0 if x["node"] == "local" else 1, x["node"]))

        entry = {
            "slug": slug,
            "name": svc["name"] if svc else slug,
            "container_status": agg["best_status"],
            # True only when EVERY instance was started outside CashPilot. With
            # a mix, the managed one can still be controlled, so the row keeps
            # its buttons and the per-instance flag marks the odd one out.
            "unmanaged": bool(agg["instances"]) and all(i.get("deployed_by") == "external" for i in agg["instances"]),
            # None, not 0.0, when CashPilot has never read this service.
            #
            # A missing earnings row means "no reading", and rendering it as
            # $0.00 told the user a service was running and earning nothing —
            # when the truth was that nothing had ever looked. Neither
            # suppressor fires in that state: collector_disconnected needs an
            # alert (none was raised, because no collector ran) and
            # collector_needs_setup only checks whether config keys are filled.
            # The flatline detector cannot catch it either: it skips rows whose
            # max balance is 0.
            #
            # This is the same defect filed three times by the audit
            # (CashPilot-vp6, -ikh, -7qk) from three different areas.
            "balance": balance_map.get(slug),
            "balance_known": slug in balance_map,
            "currency": currency_map.get(slug, "USD"),
            # None when NOTHING could be measured. A row whose every instance
            # failed its stats read must not show 0.00 — that is the same
            # fabricated idle as the per-instance case. When some instances were
            # readable the total is real, and stats_unknown says how many were
            # left out of it.
            "cpu": f"{agg['total_cpu']:.2f}" if agg.get("measured") else None,
            "memory": f"{agg['total_mem']:.1f} MB" if agg.get("measured") else None,
            "stats_unknown": agg.get("unknown_stats", 0),
            "image": agg["image"],
            "category": agg["instances"][0].get("category", ""),
            "health_score": health.get("score"),
            "uptime_pct": health.get("uptime_pct"),
            "restarts_7d": health.get("restarts", 0),
            "crashes_7d": health.get("crashes", 0),
            # "unstable" flags a service that has crashed repeatedly in the health window
            # so the dashboard can surface it at a glance (not just via the score number).
            "unstable": health.get("crashes", 0) >= 3,
            "instances": len(agg["instances"]),
            "instance_details": instance_details,
            "collector_disconnected": slug in alert_slugs and slug not in _account_disabled,
            "collector_needs_setup": slug not in alert_slugs and _collector_needs_setup(slug, config),
            # The provider turned the account off: one cause for every box and
            # for the missing balance, shown as itself (_account_disabled_check).
            "account_disabled": slug in _account_disabled,
            "collector_notice": notice_map.get(slug, ""),
            # True when the running container's image no longer matches the catalog
            # (provider migrated / re-pinned) — the dashboard prompts a re-deploy so a
            # retired image doesn't keep looking healthy while it silently stops earning.
            "image_outdated": False,
        }
        _apply_service_meta(entry, svc)
        if svc:
            # Each instance against the image it was deployed from: the catalog's,
            # or the one the operator chose for that worker, which is not drift.
            catalog_image = (svc.get("docker") or {}).get("image", "")
            entry["image_outdated"] = any(
                _image_outdated(inst.get("image", ""), custom_images.get(inst.get("_worker_id")) or catalog_image)
                for inst in agg["instances"]
            )
        elif slug in customs:
            # The user's own service: named and categorised from its definition
            # (the container's category label is the worker's default), no
            # collector, and outdated when the definition's image was edited.
            definition = customs[slug]
            entry["name"] = definition.get("name") or slug
            entry["category"] = "custom"
            entry["custom"] = True
            entry["image_outdated"] = any(
                _custom_image_outdated(inst.get("image", ""), definition.get("image", "")) for inst in agg["instances"]
            )
        result.append(entry)

    # Include external services (no Docker container, e.g. Grass, Bytelixir)
    seen_slugs = {r["slug"] for r in result}
    deployments = await database.get_deployments()
    for d in deployments:
        slug = d["slug"]
        if slug in seen_slugs:
            continue
        if d.get("status") != "external":
            continue
        svc = catalog.get_service(slug)
        health = health_map.get(slug, {})
        entry = {
            "slug": slug,
            "name": svc["name"] if svc else slug,
            "container_status": "external",
            # None, not 0.0, when CashPilot has never read this service.
            #
            # A missing earnings row means "no reading", and rendering it as
            # $0.00 told the user a service was running and earning nothing —
            # when the truth was that nothing had ever looked. Neither
            # suppressor fires in that state: collector_disconnected needs an
            # alert (none was raised, because no collector ran) and
            # collector_needs_setup only checks whether config keys are filled.
            # The flatline detector cannot catch it either: it skips rows whose
            # max balance is 0.
            #
            # This is the same defect filed three times by the audit
            # (CashPilot-vp6, -ikh, -7qk) from three different areas.
            "balance": balance_map.get(slug),
            "balance_known": slug in balance_map,
            "currency": currency_map.get(slug, "USD"),
            "cpu": "",
            "memory": "",
            "image": "",
            "category": svc.get("category", "") if svc else "",
            "health_score": None,
            "uptime_pct": None,
            "restarts_7d": 0,
            "crashes_7d": 0,
            "unstable": False,
            "instances": 0,
            "instance_details": [],
            "collector_disconnected": slug in alert_slugs and slug not in _account_disabled,
            "collector_needs_setup": slug not in alert_slugs and _collector_needs_setup(slug, config),
            # The provider turned the account off: one cause for every box and
            # for the missing balance, shown as itself (_account_disabled_check).
            "account_disabled": slug in _account_disabled,
            "collector_notice": notice_map.get(slug, ""),
        }
        _apply_service_meta(entry, svc)
        result.append(entry)

    return result


@app.get("/api/services/available")
async def api_services_available(request: Request) -> list[dict[str, Any]]:
    """Return available services from catalog, enriched with deployment status."""
    _require_auth_api(request)
    services = catalog.get_services()
    deployments = await database.get_deployments()
    deployed_slugs = {d["slug"] for d in deployments}
    # Imported here rather than at module scope: app.collectors pulls in every
    # collector module.
    from app.collectors import COLLECTOR_MAP as collector_map

    # Also check worker containers for deployed status (catches externally-deployed services)
    worker_containers = await _get_all_worker_containers()
    worker_slugs: set[str] = set()
    worker_node_counts: dict[str, set[str]] = {}
    for c in worker_containers:
        slug = c.get("slug", "")
        if slug:
            worker_slugs.add(slug)
            node = c.get("_node", "unknown")
            if slug not in worker_node_counts:
                worker_node_counts[slug] = set()
            worker_node_counts[slug].add(node)

    available = []
    for svc in services:
        if svc.get("status") in _UNDEPLOYABLE_STATUSES:
            continue  # Known non-functional — hide completely
        docker_conf = svc.get("docker", {})
        has_image = bool(docker_conf and docker_conf.get("image"))
        slug = svc.get("slug", "")
        svc["deployed"] = slug in deployed_slugs or slug in worker_slugs
        svc["manual_only"] = not has_image
        svc["node_count"] = len(worker_node_counts.get(slug, set()))
        # The setup wizard reads this endpoint, and it needs to know whether
        # earnings tracking takes a SECOND set of credentials — the service
        # detail view already says so, and the wizard is the screen a new user
        # actually sees (CashPilot-p6s).
        svc["has_collector"] = slug in collector_map
        # The grade and its top cons, computed from the published criteria.
        svc["rating_result"] = ratings.for_service(svc)
        available.append(svc)
    return available


@app.get("/api/services/{slug}")
async def api_get_service(request: Request, slug: str) -> dict[str, Any]:
    user = _require_auth_api(request)
    svc = catalog.get_service(slug) or await _custom_service(slug, reveal=False)
    if not svc:
        raise HTTPException(status_code=404, detail=f"Service '{slug}' not found")
    if svc.get("custom") and not auth.require_role(user, "owner"):
        # Plain-text variables and the command are the owner's, as /api/config is.
        svc = custom_services.redacted(svc)

    # Enrich with deployment status (same logic as /api/services/available)
    deployments = await database.get_deployments()
    deployed_slugs = {d["slug"] for d in deployments}
    worker_containers = await _get_all_worker_containers()
    worker_slugs = {c["slug"] for c in worker_containers if c.get("slug")}
    worker_nodes: set[str] = set()
    for c in worker_containers:
        if c.get("slug") == slug:
            worker_nodes.add(c.get("_node", "unknown"))

    svc["deployed"] = slug in deployed_slugs or slug in worker_slugs
    svc["node_count"] = len(worker_nodes)

    # Flag whether earnings tracking uses separate credentials (entered in
    # Settings → Collectors), so the deploy UI can tell users the container
    # credentials alone won't populate the in-dashboard balance.
    from app.collectors import COLLECTOR_MAP

    svc["has_collector"] = slug in COLLECTOR_MAP
    # The deploy form's Instances box: its ceiling, and why it is 1 when it is.
    svc["max_instances"], svc["max_instances_reason"] = scaling_limit(svc)
    # Worker id -> the image chosen instead of the catalog's, for the deploy
    # form's Image box and the custom-image badge. Absent workers use the catalog.
    svc["custom_images"] = {str(wid): image for wid, image in (await database.get_custom_images(slug)).items()}
    # Only catalog services are graded; a service a user added is not reviewed.
    svc["rating_result"] = ratings.for_service(None if svc.get("custom") else svc)
    return svc


# ---------------------------------------------------------------------------
# API: Custom services (app/custom_services.py)
# ---------------------------------------------------------------------------


async def _custom_service(slug: str, *, reveal: bool) -> dict[str, Any] | None:
    """The catalog-shaped dict of a custom service, or None.

    ``reveal=True`` carries decrypted env values and is for api_deploy only;
    nothing built with it may reach a response.
    """
    if not custom_services.is_custom_slug(slug):
        return None
    definition = await database.get_custom_service(slug)
    if definition is None:
        return None
    return custom_services.as_service(slug, definition, reveal=reveal)


class CustomServiceBody(BaseModel):
    name: str
    image: str
    env: list[dict[str, Any]] = []
    command: str = ""
    data_path: str = ""
    mem_limit: str = ""


async def _custom_deployed_on(slug: str) -> list[str]:
    """Where a custom service still runs or is recorded: box names, or ids."""
    workers = await database.list_workers()
    names = {r["name"] for r in placement.runs_on(workers).get(slug, [])}
    by_id = {w.get("id"): str(w.get("name") or w.get("id")) for w in workers}
    for row in await database.list_deployment_placements():
        if row["slug"] == slug:
            names.add(by_id.get(row["worker_id"], f"worker {row['worker_id']}"))
    if not names and await database.get_deployment(slug) is not None:
        names.add("a box CashPilot has a record for")
    return sorted(names)


@app.get("/api/custom-services")
async def api_list_custom_services(request: Request) -> list[dict[str, Any]]:
    """Every custom service, secrets masked.

    Non-secret values and the command are the owner's alone, as /api/config is:
    a viewer or a writer gets the variable names only, and the read-only token
    (_require_auth_api refuses it) gets nothing.
    """
    user = _require_auth_api(request)
    owner = auth.require_role(user, "owner")
    out = []
    for row in await database.list_custom_services():
        svc = custom_services.as_service(row["slug"], row["definition"], reveal=False)
        out.append(svc if owner else custom_services.redacted(svc))
    return out


@app.post("/api/custom-services")
async def api_create_custom_service(
    request: Request, body: CustomServiceBody, _auth: dict[str, Any] = Depends(_require_owner)
) -> dict[str, Any]:
    try:
        definition, warnings = custom_services.validate(body.model_dump())
    except ValueError as exc:
        raise HTTPException(status_code=400, detail=str(exc)) from None
    slug = custom_services.make_slug(definition["name"], await database.custom_service_slugs())
    await database.save_custom_service(slug, definition)
    logger.info("Custom service %s added from %s", slug, definition["image"])
    return {"slug": slug, "warnings": warnings}


@app.put("/api/custom-services/{slug}")
async def api_update_custom_service(
    request: Request, slug: str, body: CustomServiceBody, _auth: dict[str, Any] = Depends(_require_owner)
) -> dict[str, Any]:
    previous = await database.get_custom_service(slug) if custom_services.is_custom_slug(slug) else None
    if previous is None:
        raise HTTPException(status_code=404, detail=f"Custom service '{slug}' not found")
    try:
        definition, warnings = custom_services.validate(body.model_dump(), previous)
    except ValueError as exc:
        raise HTTPException(status_code=400, detail=str(exc)) from None
    if definition["data_path"]:
        # A data path allows one copy per box (scaling_limit), and a redeploy
        # reuses each box's recorded copy count, so saving it while a box runs
        # several would leave every redeploy refused.
        for row in await database.list_deployment_placements():
            if row["slug"] != slug:
                continue
            recorded = await database.get_deployment_spec(slug, worker_id=row["worker_id"])
            if int((recorded or {}).get("instances") or 1) > 1:
                raise HTTPException(
                    status_code=409,
                    detail=(
                        f"{definition['name']} runs several copies on a box, and a service with a data path "
                        "runs one per box. Scale it to one copy on every box first, then add the data path."
                    ),
                )
    await database.save_custom_service(slug, definition)
    logger.info("Custom service %s updated (image %s)", slug, definition["image"])
    # A running container keeps the old definition until it is redeployed.
    return {"warnings": warnings, "deployed": bool(await _custom_deployed_on(slug))}


@app.delete("/api/custom-services/{slug}")
async def api_delete_custom_service(
    request: Request, slug: str, _auth: dict[str, Any] = Depends(_require_owner)
) -> dict[str, str]:
    """Delete a definition, refused while any box runs or records it, or a deploy of it is in flight."""
    # Before any await, so no deploy can start between this check and the delete.
    if _custom_deploying.get(slug):
        raise HTTPException(status_code=409, detail=f"{slug} is being deployed right now. Try again when it is done.")
    _custom_deleting[slug] += 1
    try:
        if not custom_services.is_custom_slug(slug) or await database.get_custom_service(slug) is None:
            raise HTTPException(status_code=404, detail=f"Custom service '{slug}' not found")
        where = await _custom_deployed_on(slug)
        if where:
            raise HTTPException(
                status_code=409,
                detail=f"{slug} is still deployed on {', '.join(where)}. Remove it from every box first.",
            )
        await database.delete_custom_service(slug)
    finally:
        _custom_deleting[slug] -= 1
        if not _custom_deleting[slug]:
            del _custom_deleting[slug]
    logger.info("Custom service %s deleted", slug)
    return {"status": "deleted"}


# ---------------------------------------------------------------------------
# API: Container management
# ---------------------------------------------------------------------------


@app.get("/api/status")
async def api_status(request: Request) -> list[dict[str, Any]]:
    """Return container statuses from all workers."""
    _require_auth_api(request)
    return await _get_all_worker_containers()


class DeployRequest(BaseModel):
    env: dict[str, str] = {}
    hostname: str | None = None
    # How many copies of the service this worker runs. None (left out) keeps
    # the number this worker's deployment already has, 1 for a new one.
    instances: int | None = None
    # The image to run instead of the catalog's. None (the field left out) keeps
    # whatever this worker's deployment already uses, so a redeploy from a form
    # that never showed the box cannot drop a choice. "" goes back to the
    # catalog image; anything else must be a plain image reference.
    image: str | None = None


# repository[:tag][@sha256:digest]; see custom_services.IMAGE_REF.
_IMAGE_REF = custom_services.IMAGE_REF


def _custom_image(requested: str | None, recorded: dict[str, Any] | None) -> str:
    """The image this deploy uses instead of the catalog's, or "" for the catalog's.

    An image the operator typed wins; a box left out of the request keeps the
    one this worker's record already has. A typed value that is not an image
    reference is refused before any worker is contacted.
    """
    if requested is None:
        kept = (recorded or {}).get("custom_image")
        return kept if isinstance(kept, str) else ""
    image = requested.strip()
    if image and (len(image) > custom_services.IMAGE_MAX_LEN or not _IMAGE_REF.fullmatch(image)):
        raise HTTPException(status_code=400, detail="Image is not a valid image reference (repository[:tag][@digest])")
    return image


def _instance_param(instance: int) -> int:
    if not 1 <= instance <= MAX_INSTANCES:
        raise HTTPException(status_code=400, detail=f"instance must be between 1 and {MAX_INSTANCES}")
    return instance


def scaling_limit(svc: dict[str, Any]) -> tuple[int, str]:
    """How many copies of ``svc`` one worker may run, and why not more.

    One per worker where the provider allows one device per IP (a second copy
    behind the same address earns nothing), where the service has host
    volumes or published ports (copies would share the directories and fight
    over the ports), and where a typed device id (``device_link``) names the
    device, since every copy would carry the same one. Otherwise the ceiling is MAX_INSTANCES. The reason is ""
    when more than one is allowed. The deploy form shows the same answer.
    """
    docker_conf = svc.get("docker") or {}
    name = svc.get("name") or svc.get("slug") or "This service"
    if egress.devices_per_ip_limit(svc) == 1:
        return 1, f"{name} allows one device per IP address, so a second copy on one machine earns nothing."
    if docker_conf.get("volumes"):
        return 1, f"{name} keeps data in a volume or host directory, which two copies would share."
    if docker_conf.get("ports"):
        return 1, f"{name} publishes a port on the host, which two copies cannot share."
    link = svc.get("device_link")
    if isinstance(link, dict) and link.get("id_env"):
        # Copies get the same env, so they would all report one device id.
        label = next(
            (v.get("label") or link["id_env"] for v in docker_conf.get("env", []) if v.get("key") == link["id_env"]),
            link["id_env"],
        )
        return 1, f"{name} knows each device by its {label}, which two copies would share."
    return MAX_INSTANCES, ""


def _instance_spec(spec: dict[str, Any], instance: int, docker_conf: dict[str, Any]) -> dict[str, Any]:
    """The spec of copy ``instance``: the first copy's, under its own names.

    The env values the catalog derives from {hostname} are device names the
    provider counts by, so each copy gets its own (cashpilot-box-2), and so are
    the ids it marks ``per_device``: each copy gets its own, derived from the
    original and the copy number as CashPilot Desktop does. Nothing else
    differs: same image, credentials, capabilities and limits. Raises
    ValueError when a device id cannot be derived: the copy is refused.
    """
    named = {var["key"] for var in docker_conf.get("env", []) if "{hostname}" in str(var.get("default", ""))}
    env = {
        key: (f"{value}-{instance}" if key in named and value else value)
        for key, value in (spec.get("env") or {}).items()
    }
    rename = proxies.device_id_renamer(env, docker_conf, str(instance))
    env = {key: rename(value) for key, value in env.items()}
    copy = {**spec, "env": env, "instance": instance, "moved_mounts": []}
    copy["hostname"] = f"{spec['hostname']}-{instance}" if spec.get("hostname") else None
    # A command that carries a device name (Traffmonetizer's --device-name) is
    # filled again from this copy's env, or every copy registers as the first.
    if spec.get("command") and docker_conf.get("command"):
        copy["command"] = re.sub(r"\$\{(\w+)\}", lambda m: env.get(m.group(1), m.group(0)), docker_conf["command"])
    elif spec.get("command"):
        copy["command"] = rename(spec["command"])
    return copy


@app.post("/api/deploy/{slug}")
async def api_deploy(
    request: Request,
    slug: str,
    body: DeployRequest,
    worker_id: int | None = None,
    _auth: dict[str, Any] = Depends(_require_owner),
) -> dict[str, Any]:
    """Validate the effective service configuration and deploy it to the selected worker."""
    if not custom_services.is_custom_slug(slug):
        return await _deploy(slug, body, worker_id)
    # A custom deploy holds its slug until it is recorded, so a delete cannot
    # remove the definition while the worker is still pulling and starting it,
    # which would leave a container running that no definition describes.
    # Checked and taken with no await in between: one event loop, one UI process.
    if slug in _custom_deleting:
        raise HTTPException(status_code=409, detail=f"{slug} is being deleted")
    _custom_deploying[slug] += 1
    try:
        return await _deploy(slug, body, worker_id)
    finally:
        _custom_deploying[slug] -= 1
        if not _custom_deploying[slug]:
            del _custom_deploying[slug]


#: Custom slugs with deploys in flight, and with deletes in flight (how many of each).
#: See api_deploy and api_delete_custom_service.
_custom_deploying: collections.Counter[str] = collections.Counter()
_custom_deleting: collections.Counter[str] = collections.Counter()


async def _deploy(slug: str, body: DeployRequest, worker_id: int | None) -> dict[str, Any]:
    # dict[str, Any], not dict[str, str]: kept_from_previous_deployment is a
    # list, and FastAPI validates the response against this annotation AFTER
    # the deploy has run and persisted — a stricter type turns the successful
    # deployments that had something to report into 500s.
    worker_id = await _resolve_worker_id(worker_id)
    svc = catalog.get_service(slug) or await _custom_service(slug, reveal=True)
    if not svc:
        raise HTTPException(status_code=404, detail=f"Service '{slug}' not found")
    # A custom service is deployed from its stored definition and nothing else:
    # the image and the variables are edited there, so a deploy that tries to
    # change them is refused rather than silently ignored.
    is_custom = bool(svc.get("custom"))
    if is_custom:
        if (body.image or "").strip():
            raise HTTPException(status_code=400, detail="Change the image by editing the custom service")
        if any(str(v).strip() for v in (body.env or {}).values()):
            raise HTTPException(status_code=400, detail="Edit the custom service to change its variables")
    status = svc.get("status")
    if status in _UNDEPLOYABLE_STATUSES:
        # 410 for a service that is permanently gone, 409 for one that is merely
        # broken right now and may come back. Both are hidden from the catalog
        # listing, so reaching here means a direct link, a stale page or an API client.
        raise HTTPException(
            status_code=409 if status == "broken" else 410,
            detail=f"Service '{slug}' is no longer available for deployment ({status})",
        )

    docker_conf = svc.get("docker", {})
    image = docker_conf.get("image")
    if not image:
        raise HTTPException(status_code=400, detail=f"Service '{slug}' has no Docker image")

    # Build full env: YAML defaults + user overrides + {hostname} substitution.
    #
    # The substitution runs LAST, over the merged result, because it used to run
    # only over the defaults — and the setup wizard prefills each input with the
    # raw default, so the browser posts "cashpilot-{hostname}" back as a USER
    # value. A user value wins over the default, so the placeholder was never
    # substituted and the literal string shipped as the device name. Providers
    # count devices by that name, so every host deployed through the wizard
    # registered under one identical name.
    #
    # Applying it to overrides too is also the right reading of a value someone
    # typed by hand: nobody means the eight characters "{hostname}".
    hn = body.hostname or HOSTNAME_PREFIX
    env: dict[str, str] = {}
    for var in docker_conf.get("env", []):
        env[var["key"]] = str(var.get("default", ""))
    # A box left blank is not a value. The form posts "" for every field the
    # operator did not fill, including the {hostname} templates it shows as a
    # placeholder precisely so the server fills them in - and "" overriding the
    # default sent an empty device name on a first deploy.
    env.update({key: str(value).strip() for key, value in (body.env or {}).items() if str(value).strip()})
    env = {k: v.replace("{hostname}", hn) if isinstance(v, str) else v for k, v in env.items()}

    # What this service was ACTUALLY deployed with, if anything. Loaded before
    # the required-field check because a redeploy must not demand values the
    # deployment already has: rejecting here would mean the operator has to
    # retype Storj's IDENTITY_DIR/STORAGE_DIR on every redeploy, which is
    # precisely what recording the spec exists to avoid.
    # THIS worker's record: another machine's carries another machine's env.
    recorded = await database.get_deployment_spec(slug, worker_id=worker_id)
    recorded_env = (recorded or {}).get("env") or {}
    custom_image = "" if is_custom else _custom_image(body.image, recorded)

    # What the operator actually TYPED on this deploy. The dashboard has no
    # separate redeploy action: it posts the deploy form, and the form posts
    # every field - the ones left blank, and the ones still showing the catalog
    # default. Neither is a decision. Treating them as one let an empty box
    # overwrite a recorded credential and let a prefilled path count as the
    # operator relocating a mount, which is how a redeploy moves a node off the
    # directory that holds its identity.
    catalog_defaults = {var["key"]: str(var.get("default", "")) for var in docker_conf.get("env", [])}
    # Stripped once and used stripped: "/mnt/s " is not a different path from
    # "/mnt/s", and sent as typed it becomes a host directory with a space in it.
    posted_env = {key: str(value).strip() for key, value in (body.env or {}).items()}
    typed_env = {
        key: value for key, value in posted_env.items() if value and value != catalog_defaults.get(key, "").strip()
    }

    # Validate required env vars are not blank.
    missing = [
        var.get("label", var["key"])
        for var in docker_conf.get("env", [])
        if var.get("required")
        and not env.get(var["key"], "").strip()
        and not str(recorded_env.get(var["key"], "")).strip()
    ]
    if missing:
        raise HTTPException(status_code=400, detail=f"Missing required fields: {', '.join(missing)}")

    # How many copies this worker runs, decided before anything is deployed.
    instances = body.instances if body.instances is not None else int((recorded or {}).get("instances") or 1)
    _instance_param(instances)
    # Read only when copies are involved, so a plain deploy asks the worker
    # record nothing it did not ask before.
    running_copies = (
        await _worker_instances(worker_id, slug)
        if body.instances is not None or (recorded or {}).get("instances")
        else set()
    )
    if instances > 1:
        limit, why = scaling_limit(svc)
        if instances > limit:
            raise HTTPException(status_code=400, detail=why)
        if "instances" not in await _worker_features(worker_id):
            raise HTTPException(
                status_code=409,
                detail="This worker is too old to run a service more than once. Update the worker first.",
            )

    # Ports — key is "container_port/protocol" per Docker SDK. A mapping with a
    # host address ("127.0.0.1:9980:9980") becomes [address, port]: a web UI
    # that controls a wallet must not be published on every interface. The
    # worker turns the pair into the (address, port) tuple Docker expects.
    ports: dict[str, int | list[str | int]] = {}
    for mapping in docker_conf.get("ports", []):
        try:
            key, binding = catalog.parse_port(mapping)
        except ValueError as exc:
            # The loader rejects these, so only a hand-edited cache gets here.
            raise HTTPException(status_code=400, detail=f"Invalid catalog port for {slug}: {exc}") from exc
        ports[key] = binding

    # Volumes: resolve ${VAR} in host paths using env
    volumes: dict[str, dict[str, str]] = {}
    for mapping in docker_conf.get("volumes", []):
        if ":" in str(mapping):
            parts = str(mapping).split(":")
            host_path = re.sub(r"\$\{(\w+)\}", lambda m: env.get(m.group(1), m.group(0)), parts[0])
            container_path = parts[1]
            mode = parts[2] if len(parts) > 2 else "rw"
            volumes[host_path] = {"bind": container_path, "mode": mode}

    spec: dict[str, Any] = {
        "image": image,
        "env": env,
        "hostname": body.hostname,
        "ports": ports,
        "volumes": volumes,
        "network_mode": docker_conf.get("network_mode") or None,
        "cap_add": docker_conf.get("cap_add") or None,
        # Some services need a device, not just a capability: Mysterium cannot
        # carry wireguard traffic without /dev/net/tun, and without it the node
        # starts, registers and earns nothing.
        "devices": docker_conf.get("devices") or None,
        "privileged": docker_conf.get("privileged", False),
    }

    # The command template; its ${VAR} placeholders are filled in below, AFTER
    # the recorded spec is merged, so they see the env this deployment really has.
    raw_command = docker_conf.get("command") or None

    # Durable resource limits (mem_limit / mem_reservation / oom_score_adj /
    # cpu_shares), declared in the service YAML. Only forwarded when present.
    resources = docker_conf.get("resources")
    if resources:
        spec["resources"] = resources

    # An existing deployment is rebuilt from what it actually ran, not from the
    # catalog - see _merge_recorded_spec. `recorded` was loaded above, before the
    # required-field check.
    divergence: list[str] = []
    # Which env vars feed which mount, so a relocation applies only to the
    # mount it actually names.
    keys_by_target: dict[str, set[str]] = {}
    for mapping in docker_conf.get("volumes", []):
        raw = str(mapping)
        if ":" not in raw:
            continue
        host_part, target = raw.split(":")[0], raw.split(":")[1]
        keys_by_target.setdefault(target, set()).update(m.group(1) for m in re.finditer(r"\$\{(\w+)\}", host_part))
    # Not for a custom service: its definition is the only source, so an edit
    # to it always reaches the redeploy instead of losing to the record.
    if recorded and not is_custom:
        spec, divergence = _merge_recorded_spec(spec, recorded, typed_env, keys_by_target)

    # The command always comes from the catalog, filled from the MERGED env.
    #
    # Nobody types a command: the form has no field for one, so a recorded
    # command is only ever an older catalog template filled with that
    # deployment's values. Keeping it protected no decision and froze two
    # things that must change. A catalog fix to the command never reached a
    # running service (Mysterium kept publishing its UI and API on 0.0.0.0 after
    # the catalog bound them to loopback), and a password retyped on redeploy
    # never reached a service that takes it on the command line (Honeygain,
    # IPRoyal, Traffmonetizer), because the old command still carried the old one.
    #
    # Filling it from the merged env is what makes that safe: a blank redeploy
    # gets the recorded credentials, where the pre-merge env had only the form's
    # values and left the placeholders unfilled, which is why the record had to
    # win before.
    merged_env = spec.get("env") or {}
    # A value the catalog constrains must match before anything is deployed:
    # it may land on the command line, where "127.0.0.1, 10.0.0.1" becomes two
    # arguments and the service refuses to start. Checked on the MERGED env,
    # so a recorded value is held to a constraint added after it was stored
    # (a Repocket password saved as RP_API_KEY would otherwise ride every blank
    # redeploy into another restart loop). A blank optional value is not a
    # value, so it is not checked. The rejected value is never echoed: it may
    # be a credential.
    invalid = [
        var.get("label", var["key"])
        for var in docker_conf.get("env", [])
        if (
            var.get("pattern")
            and str(merged_env.get(var["key"], ""))
            and not re.fullmatch(var["pattern"], str(merged_env.get(var["key"], "")))
        )
        or (var.get("port_range") and not _is_port_range(str(merged_env.get(var["key"], ""))))
    ]
    if invalid:
        raise HTTPException(status_code=400, detail=f"Invalid value for: {', '.join(invalid)}")
    if catalog.fence_gated(svc):
        spec = await _fence_gated_spec(spec, docker_conf, merged_env, worker_id, slug, divergence, custom_image)

    if raw_command:
        spec["command"] = re.sub(r"\$\{(\w+)\}", lambda m: merged_env.get(m.group(1), m.group(0)), raw_command)
    else:
        spec.pop("command", None)
    if is_custom:
        # What the worker labels the container with, so it reads as user-supplied
        # wherever it shows up. _instance_spec carries it to every copy.
        spec["labels"] = {constants.LABEL_CUSTOM: "true"}
    elif recorded and "command" in recorded and recorded.get("command") != spec.get("command"):
        # Never the command itself: it carries credentials.
        divergence.append("command: using the catalog's current command; this service last ran a different one")
    # The image is the catalog's unless the operator chose another; the choice is
    # recorded with the spec, so the next redeploy on this worker keeps it.
    # Capabilities, devices, the entrypoint and the stop timeout still come from
    # the worker's own catalog entry, so a different image gains nothing the
    # service does not already have.
    if custom_image:
        spec["image"] = custom_image
        spec["custom_image"] = custom_image
    else:
        spec.pop("custom_image", None)
    if divergence:
        logger.info("Redeploying %s from its recorded spec: %s", slug, "; ".join(divergence))

    # The worker keeps every mount of the container it replaces where it is now,
    # except the ones named here: a path the operator typed on THIS deploy is a
    # move they asked for. The record above cannot settle this on its own - it
    # is one row per service for the whole fleet, and a container made or edited
    # by hand has no record at all - so the running container has the last word.
    moved_mounts = sorted(target for target, keys in keys_by_target.items() if keys & set(typed_env))

    # Every extra copy must be able to get its own device id before anything is
    # replaced, or the first copy would be redeployed and the rest refused.
    for n in range(2, instances + 1):
        try:
            _instance_spec(spec, n, docker_conf)
        except ValueError as exc:
            raise HTTPException(status_code=400, detail=f"{svc.get('name') or slug} copy {n}: {exc}") from exc

    result = await _proxy_worker_deploy(worker_id, slug, {**spec, "moved_mounts": moved_mounts})
    container_id = result.get("container_id", "remote")
    # Reported, deliberately NOT recorded. The worker reads the running container
    # on every deploy, so the record does not need the kept source - and writing
    # it there made the next redeploy send a host path the worker's bind-path
    # rules refuse (403), and handed one worker's path to every other worker,
    # since the record is one row per service for the whole fleet.
    raw_kept = result.get("kept_mounts")
    for k in raw_kept if isinstance(raw_kept, list) else []:
        if isinstance(k, dict):
            divergence.append(
                f"mounts: {k.get('target')} stays on {k.get('kept')}, where the running container keeps it, "
                f"instead of {k.get('requested')}"
            )
    if instances > 1:
        spec["instances"] = instances
    else:
        spec.pop("instances", None)
    await database.save_deployment(slug=slug, container_id=container_id, spec=spec, worker_id=worker_id)
    # The extra copies, after the record: if one fails, the next redeploy still
    # knows how many this worker should run and tries again.
    for n in range(2, instances + 1):
        await _proxy_worker_deploy(worker_id, slug, _instance_spec(spec, n, docker_conf))
    for n in sorted(c for c in running_copies if c > instances):
        await _proxy_to_worker(worker_id, "DELETE", f"/api/containers/{slug}/instances/{n}")
    await database.record_health_event(slug, "start", f"deployed to worker {worker_id}")
    metrics.record_container_lifecycle("deploy", slug)
    _spawn(_run_collection())
    response: dict[str, Any] = {"status": "deployed", "container_id": container_id}
    # From the merged env: a redeploy with a blank form still runs the recorded id.
    if link := catalog.device_link_step(svc, merged_env):
        registered = await _register_device(slug, link["id"], merged_env)
        link["registered"] = registered is True
        link["retrying"] = registered is None
        response["device_link"] = link
    if divergence:
        response["kept_from_previous_deployment"] = divergence
    return response


_is_port_range = catalog.is_port_range


def fence_gated_ports(docker_conf: dict[str, Any], env: dict[str, Any]) -> dict[str, Any]:
    """The ports of a fence-gated service, always from the catalog and the env.

    The catalog's static ports, the WebUI port on each address ``ui_port.env``
    lists (one binding, or several), and the UDP range ``udp_port_range.env``
    names, published 1:1. Raises ValueError naming what is wrong.
    """
    ports: dict[str, Any] = {}
    for mapping in docker_conf.get("ports") or []:
        key, binding = catalog.parse_port(mapping)
        ports[key] = binding
    ui = docker_conf.get("ui_port")
    if isinstance(ui, dict):
        bindings = catalog.ui_bindings(str(env.get(ui["env"]) or "127.0.0.1"), int(ui["port"]))
        # One binding travels as [address, port], which every worker reads; a
        # list of them only to a worker that can publish one port several times.
        ports[f"{int(ui['port'])}/tcp"] = bindings[0] if len(bindings) == 1 else bindings
    rng = docker_conf.get("udp_port_range")
    if isinstance(rng, dict):
        ports.update(catalog.expand_udp_range(str(env.get(rng["env"]) or ""), int(rng.get("max_ports") or 1000)))
    return ports


async def _fence_gated_spec(
    spec: dict[str, Any],
    docker_conf: dict[str, Any],
    env: dict[str, Any],
    worker_id: int,
    slug: str,
    divergence: list[str],
    custom_image: str,
) -> dict[str, Any]:
    """Where a fence-gated service's network namespace lives is the catalog's alone.

    The record wins for an ordinary service's network mode and ports, so a
    redeploy reproduces them. Here they are the fence: a node recorded on the
    host network (every Mysterium through 2.0.6) must move onto the fenced
    bridge on its next redeploy, with its ports rebuilt for it, or the redeploy
    would carry NET_ADMIN back into the host's namespace. The identity volume is
    a mount, so it is kept as before.
    """
    if custom_image:
        raise HTTPException(
            status_code=400,
            detail=f"{slug} runs only the image the catalog pins, since it holds NET_ADMIN in its own namespace",
        )
    if "fence_gated" not in await _worker_features(worker_id):
        raise HTTPException(
            status_code=409,
            detail=(
                f"This worker is too old to run {slug} behind the LAN fence. Update the worker first; the node "
                "keeps running as it is until then."
            ),
        )
    try:
        ports = fence_gated_ports(docker_conf, env)
    except ValueError as exc:
        raise HTTPException(status_code=400, detail=f"Invalid value for {slug}: {exc}") from None
    merged = dict(spec)
    # The record's network mode and ports do not win here, so saying they were kept would be false.
    divergence[:] = [d for d in divergence if not d.startswith(("network_mode: keeping", "ports: keeping"))]
    if (merged.get("network_mode") or None) != (docker_conf.get("network_mode") or None):
        divergence.append("network_mode: moving to the fenced bridge the catalog declares now")
    merged["network_mode"] = docker_conf.get("network_mode") or None
    merged["ports"] = ports
    return merged


def _merge_recorded_spec(
    catalog_spec: dict[str, Any],
    recorded: dict[str, Any],
    user_env: dict[str, str],
    volume_env_keys_by_target: dict[str, set[str]] | None = None,
) -> tuple[dict[str, Any], list[str]]:
    """Rebuild an EXISTING deployment from what it actually ran.

    The catalog describes what a NEW deployment looks like. It does not describe
    what this one looks like: the running container may sit on a bind mount
    where the catalog declares a named volume, or on a host path that only
    existed because of an env substitution. Rebuilding from the catalog silently
    produces a different container, and the worker destroys the old one before
    anything can compare them - which is how a node identity gets orphaned.

    So for an existing service the record wins, with deliberate exceptions:
    the image, the capabilities and the command always come from the catalog
    (otherwise upgrades and fixes to them could never land; the command is
    filled by the caller, from the env merged here), and anything the user
    explicitly typed on this deploy wins over the stored value (otherwise a
    credential could never be corrected).

    Returns the merged spec and a list of human-readable divergences, which are
    information for the operator rather than something to resolve silently.
    """
    merged = dict(catalog_spec)
    divergence: list[str] = []

    # Ports define the container's identity on the network. Reproduce them.
    stored_ports = recorded.get("ports")
    if stored_ports:
        if stored_ports != catalog_spec.get("ports"):
            divergence.append(
                "ports: keeping the ports this service was deployed with; the catalog now declares different ports"
            )
        merged["ports"] = stored_ports

    # Mounts are decided PER MOUNT, not for the volumes block as a whole.
    #
    # If the operator typed a value this deploy that the catalog substitutes into
    # a host path - Storj's ${IDENTITY_DIR}, say - they are moving that data on
    # purpose and the new path wins. Every OTHER mount is still reproduced from
    # the record: a service can have several independent path variables (Storj
    # has IDENTITY_DIR and STORAGE_DIR), and supplying one must not silently
    # reset the others.
    #
    # The trigger is the volume-substituting keys alone. Retyping a password must
    # never cost a service its mounts, which is the bug this function exists to
    # fix in the first place.
    stored_volumes = recorded.get("volumes")
    if stored_volumes:
        by_target = volume_env_keys_by_target or {}
        catalog_volumes = catalog_spec.get("volumes") or {}
        catalog_by_target = {spec.get("bind"): (host, spec) for host, spec in catalog_volumes.items()}
        recorded_targets = {spec.get("bind") for spec in stored_volumes.values()}

        merged_volumes: dict[str, Any] = {}
        moved: list[str] = []
        kept = False

        for host, spec in stored_volumes.items():
            target = spec.get("bind")
            if set(user_env) & by_target.get(target, set()):
                new_host, new_spec = catalog_by_target.get(target, (host, spec))
                merged_volumes[new_host] = new_spec
                moved.append(target)
            else:
                merged_volumes[host] = spec
                if catalog_by_target.get(target, (host, spec)) != (host, spec):
                    kept = True

        # Mounts the catalog has added since this service was deployed.
        for target, (host, spec) in catalog_by_target.items():
            if target not in recorded_targets:
                merged_volumes[host] = spec

        if moved:
            divergence.append(f"mounts: using the paths you supplied this deploy for {', '.join(sorted(moved))}")
        if kept:
            divergence.append(
                "mounts: keeping the mounts this service was deployed with; the catalog now declares different mounts"
            )
        merged["volumes"] = merged_volumes

    # Runtime shape that the catalog may since have changed underneath a
    # running service. hostname is included because several bandwidth services
    # key their device identity to the container hostname: on a redeploy where
    # the operator leaves the hostname field empty, catalog_spec["hostname"] is
    # None, and rebuilding from that would silently give the service a new
    # identity. As with env, a hostname the operator typed THIS deploy still
    # wins - a non-empty catalog_spec value is what they just asked for.
    # command is not here: the caller rebuilds it from the catalog template and
    # the merged env, and reports the change itself (see api_deploy).
    for field in ("network_mode",):
        if field in recorded and recorded.get(field) != catalog_spec.get(field):
            divergence.append(f"{field}: keeping the deployed value")
            merged[field] = recorded[field]

    # cap_add is NOT reproduced from the record; like the image, it comes from
    # the catalog. Capabilities carry no deployment identity, the operator cannot
    # set them, and the worker only accepts what its own catalog declares for the
    # slug. Recorded-wins made a capability fix unreachable for every deployed
    # service: mysterium gained SETUID/SETGID in the catalog and a redeploy kept
    # rebuilding the container with the list that was broken. It failed the other
    # way too, since a capability the catalog had dropped came back from the record
    # and the worker refused the deploy.
    def _caps(spec: dict[str, Any]) -> set[str]:
        return {str(cap).upper() for cap in spec.get("cap_add") or []}

    if "cap_add" in recorded and _caps(recorded) != _caps(catalog_spec):
        divergence.append("cap_add: using the capabilities the catalog declares now, not the ones it was deployed with")

    # resources is merged per KEY, unlike the runtime-shape fields above, and
    # for the env reason: limits carry no deployment identity, and the catalog
    # is where NEW tuning keys land (a value change to a key the record
    # already set still keeps the recorded value, reported as a divergence).
    # Whole-dict recorded-wins made every catalog
    # resource change unreachable for a deployed service — the record predating
    # a new key (cpu_shares) beat the catalog forever, and the only escape was
    # remove-and-redeploy, which for storj is the identity-loss operation the
    # recorded spec exists to prevent. A key the record already set still wins:
    # that is the value this deployment verifiably ran with.
    stored_res = recorded.get("resources")
    if isinstance(stored_res, dict):
        catalog_res = catalog_spec.get("resources")
        merged_res = dict(catalog_res) if isinstance(catalog_res, dict) else {}
        merged_res.update(stored_res)
        if merged_res != (catalog_res or {}):
            divergence.append("resources: keeping the limits this service was deployed with")
        if merged_res:
            merged["resources"] = merged_res

    stored_hostname = recorded.get("hostname")
    if stored_hostname and not catalog_spec.get("hostname") and stored_hostname != catalog_spec.get("hostname"):
        divergence.append("hostname: keeping the hostname this service was deployed with")
        merged["hostname"] = stored_hostname

    # env: catalog defaults provide any newly added keys, the record provides
    # what this deployment actually used, and an explicit entry from the user
    # overrides both.
    stored_env = recorded.get("env")
    if isinstance(stored_env, dict):
        env = dict(catalog_spec.get("env") or {})
        for key, value in stored_env.items():
            if key not in user_env:
                env[key] = value
        merged["env"] = env

    return merged, divergence


# The stop/restart/remove lifecycle actions each have two public routes — the flat
# /api/{action}/{slug} and the service-scoped /api/services/{slug}/... — with identical
# behavior. Both sets of routes delegate to these helpers so the writer-guard, worker
# resolution, proxy, and bookkeeping live in exactly one place per action.
async def _svc_stop(request: Request, slug: str, worker_id: int | None, instance: int = 1) -> dict[str, str]:
    _require_writer(request)
    worker_id = await _resolve_worker_id(worker_id)
    result = await _proxy_worker_command(worker_id, "stop", slug, instance=instance)
    await database.record_health_event(slug, "stop")
    metrics.record_container_lifecycle("stop", slug)
    return result


async def _svc_restart(request: Request, slug: str, worker_id: int | None, instance: int = 1) -> dict[str, str]:
    _require_writer(request)
    worker_id = await _resolve_worker_id(worker_id)
    result = await _proxy_worker_command(worker_id, "restart", slug, instance=instance)
    await database.record_health_event(slug, "restart")
    metrics.record_container_lifecycle("restart", slug)
    return result


async def _svc_remove(
    request: Request,
    slug: str,
    worker_id: int | None,
    delete_volumes: bool,
    allow_delete_critical: bool = False,
) -> dict[str, Any]:
    _require_writer(request)
    if allow_delete_critical:
        # Overriding the critical-volume guard destroys state with no server-side
        # copy - a node identity, a generated wallet. Deploying already requires
        # owner; permanently destroying the money must not be the easier action.
        _require_owner(request)
    worker_id = await _resolve_worker_id(worker_id)
    params: dict[str, str] | None = None
    if delete_volumes:
        params = {"delete_volumes": "true"}
        if allow_delete_critical:
            params["allow_delete_critical"] = "true"
    result = await _proxy_worker_command(worker_id, "remove", slug, params=params)
    await database.remove_deployment(slug, worker_id=worker_id)
    # The worker removed the service's proxied copies with it (worker_api
    # remove_all), so their records go too. The removal itself already
    # happened; a failure here must not report it as failed.
    try:
        if isinstance(result, dict) and result.get("proxied_error"):
            # Some proxied copies survived; their records stay so they can be removed.
            logger.warning("Removed %s, but some proxied copies remain: %s", slug, result["proxied_error"])
        else:
            await database.delete_proxied_copies_of(worker_id, slug)
    except Exception as exc:  # noqa: BLE001
        logger.warning("Removed %s, but could not drop its proxied copy records: %s", slug, exc)
    await database.record_health_event(slug, "remove")
    metrics.record_container_lifecycle("remove", slug)
    return result


@app.post("/api/stop/{slug}")
async def api_stop(request: Request, slug: str, worker_id: int | None = None) -> dict[str, str]:
    return await _svc_stop(request, slug, worker_id)


@app.post("/api/restart/{slug}")
async def api_restart(request: Request, slug: str, worker_id: int | None = None) -> dict[str, str]:
    return await _svc_restart(request, slug, worker_id)


@app.delete("/api/remove/{slug}")
async def api_remove(
    request: Request,
    slug: str,
    worker_id: int | None = None,
    delete_volumes: bool = False,
    allow_delete_critical: bool = False,
) -> dict[str, Any]:
    return await _svc_remove(request, slug, worker_id, delete_volumes, allow_delete_critical)


@app.delete("/api/deployments/{slug}")
async def api_forget_deployment(request: Request, slug: str, worker_id: int | None = None) -> dict[str, str]:
    """Forget a deployment that no box runs: the way out of the missing alert.

    Removing a container through a worker needs the container to exist, so a
    record whose container is already gone had no way to be dropped. This only
    touches CashPilot's records, never a worker. With ``worker_id`` it forgets
    that box's record; without, every record of the service. Refused while a
    box still reports the service, or while a box holding a record of it cannot
    read Docker, so it cannot hide a running container.
    """
    _require_writer(request)
    workers = await database.list_workers()
    running = placement.runs_on(workers).get(slug, [])
    recorded_on = {s["worker_id"] for s in await database.list_deployment_placements() if s["slug"] == slug}
    targets = recorded_on if worker_id is None else {worker_id}
    blind = [w for w in workers if w.get("id") in targets and w.get("status") == "online" and placement.is_blind(w)]
    if blind:
        names = ", ".join(str(w.get("name") or w.get("id")) for w in blind)
        raise HTTPException(
            status_code=409,
            detail=f"{names} cannot read Docker right now, so CashPilot cannot tell whether {slug} runs there.",
        )
    if worker_id is None:
        if running:
            names = ", ".join(r["name"] for r in running)
            raise HTTPException(status_code=409, detail=f"{slug} still runs on {names}. Remove it there instead.")
        await database.forget_deployment(slug)
    else:
        if any(r["worker_id"] == worker_id for r in running):
            raise HTTPException(status_code=409, detail=f"{slug} still runs on that box. Remove it there instead.")
        await database.remove_deployment(slug, worker_id=worker_id)
    logger.info("Forgot the %s deployment record%s", slug, "" if worker_id is None else f" on worker {worker_id}")
    still_recorded = await database.get_deployment(slug) is not None or any(
        s["slug"] == slug for s in await database.list_deployment_placements()
    )
    if not still_recorded:
        # Nothing is left to be missing, so the alert goes now, even on a fleet
        # where the check below cannot tell (a blind Docker host elsewhere).
        await _drop_missing_alert(slug)
    await _missing_deployment_check(workers)
    return {"status": "forgotten"}


async def _drop_missing_alert(slug: str) -> None:
    global _collector_alerts
    await database.clear_alerts("missing", slug)
    _missing_bell.pop(slug, None)
    for key in [k for k in _deployment_first_missing if k[0] == slug]:
        _deployment_first_missing.pop(key, None)
    _collector_alerts = [a for a in _collector_alerts if not (a.get("kind") == "missing" and a["platform"] == slug)]


# ---------------------------------------------------------------------------
# Helpers: proxy commands / logs to worker nodes
# ---------------------------------------------------------------------------


async def _get_verified_worker_url(worker: dict[str, Any]) -> tuple[str, dict[str, str]]:
    """Validate a worker record and return (url, headers).

    When the worker URL is a hostname, the returned URL is pinned to the IP that
    passed validation and a ``Host`` header preserves the original name — closing
    the DNS-rebinding TOCTOU between validation and the httpx request.
    """
    if not worker:
        raise HTTPException(status_code=404, detail="Worker not found")
    if worker["status"] != "online":
        raise HTTPException(status_code=503, detail="Worker is offline")
    if not worker["url"]:
        raise HTTPException(status_code=503, detail="Worker URL not known")
    url, pinned_ip = await asyncio.to_thread(_validate_worker_url, worker["url"])
    host_header: str | None = None
    if pinned_ip:
        url, host_header = _pin_url_to_ip(url, pinned_ip)
    # Authenticate to the worker with ITS OWN key once enrolled; fall back to the
    # shared bootstrap key only for workers that have not enrolled yet. Post-cutover
    # an enrolled worker rejects the shared key, so the UI must present its own.
    cid = worker.get("client_id") or ""
    # Use the per-worker key only once the worker has CONFIRMED it.
    #
    # get_worker_key discards the confirmed flag, so the UI signed outbound
    # commands with a key the worker may never have received — reachable
    # whenever _save_worker_key failed on the worker ("staying on shared key").
    # The worker then read as online from its heartbeats while every deploy,
    # start, stop, restart and logs call failed 401, with nothing connecting
    # the two symptoms.
    key_state = await database.get_worker_key_state(cid) if cid else (None, False)
    stored_key, key_confirmed = key_state if isinstance(key_state, tuple) else (key_state, False)
    auth_key = stored_key if (stored_key and key_confirmed) else FLEET_API_KEY
    headers: dict[str, str] = {}
    if auth_key:
        headers["Authorization"] = f"Bearer {auth_key}"
    if host_header:
        headers["Host"] = host_header
    return url, headers


# Worker error bodies are not forwarded verbatim: they can carry internal paths
# and hostnames. Only errors the worker deliberately structures for the operator
# are passed through, and only the fields we expect.
_FORWARDABLE_WORKER_ERRORS = {"critical_volume"}


def _safe_worker_detail(resp: Any) -> dict[str, Any] | None:
    """Return a worker error detail that is safe to show, or None.

    A refusal to destroy irreplaceable data is useless to the operator without
    the reason - "409" alone does not say which volume, or what is in it - so
    that specific shape is forwarded. Everything else stays generic.
    """
    try:
        detail = (resp.json() or {}).get("detail")
    except (ValueError, AttributeError):
        return None
    error = detail.get("error") if isinstance(detail, dict) else None
    # error comes from a remote worker body; a list/dict would make the `not in`
    # test raise TypeError and turn a 409 into a 500. Require a string.
    if not isinstance(error, str):
        return None
    # The self-update refusal is the second forwardable shape: a fixed,
    # worker-authored instruction string ("not compose-managed, run these
    # commands instead"). Without this, the sanitizer replaced it with the
    # generic "Worker request failed" and the promised manual instructions
    # never reached the operator.
    # The isolation check's "already running" and "no Docker socket" answers
    # are fixed worker-authored strings too; without them the card read
    # "Worker request failed" and busy looked the same as broken.
    if error in ("self_update_unavailable", "isolation_check_busy", "isolation_check_unavailable"):
        return {"error": error, "message": str(detail.get("message") or "")}
    if error not in _FORWARDABLE_WORKER_ERRORS:
        return None
    blocked = detail.get("blocked")
    if not isinstance(blocked, list):
        return None
    return {
        "error": detail["error"],
        "message": str(detail.get("message") or ""),
        "hint": str(detail.get("hint") or ""),
        "blocked": [
            {
                "volume": str(b.get("volume", "")),
                "target": str(b.get("target", "")),
                "holds": str(b.get("holds", "")),
            }
            for b in blocked
            if isinstance(b, dict)
        ],
    }


# A plain-string worker refusal is shown only for the statuses where the worker
# writes the text itself: its spec checks (400/403: blocked capabilities, a
# volume outside the allowed roots, a bad memory limit), the missing copy (404),
# the operator settings it cannot use (409: network, runtime, an init-system
# image) and the daemon it cannot reach (503). A 500 is an unhandled failure
# whose text nobody vetted, and a 422 is FastAPI echoing the request body
# (env values included), so both stay generic.
_FORWARDABLE_WORKER_TEXT_STATUSES = {400, 401, 403, 404, 409, 503}
_WORKER_TEXT_MAX = 400
# Env values shorter than this are not masked: a PUID of 1000 or a mode of
# "host" would otherwise garble the worker's own numbers and words, and no
# provider credential is that short.
_MASK_MIN_LEN = 8
# FastAPI's own answer for a route the worker does not have, which says less
# than the status does.
_FASTAPI_NOT_FOUND = "Not Found"


def _worker_refusal_text(resp: Any, sent: dict[str, Any] | None) -> str | None:
    """The worker's own one-line refusal reason, cleaned for display, or None."""
    if resp.status_code not in _FORWARDABLE_WORKER_TEXT_STATUSES:
        return None
    try:
        detail = (resp.json() or {}).get("detail")
    except (ValueError, AttributeError):
        return None
    if not isinstance(detail, str) or detail == _FASTAPI_NOT_FOUND:
        return None
    text = detail
    # The worker never echoes env values in these messages, but the deploy spec
    # carries live credentials, so a value that slipped in is masked anyway.
    env = sent.get("env") if isinstance(sent, dict) else None
    if isinstance(env, dict):
        for value in sorted((str(v) for v in env.values() if v is not None), key=len, reverse=True):
            if len(value) >= _MASK_MIN_LEN:
                text = text.replace(value, "***")
    text = " ".join(text.split())  # one line: newlines and tabs collapse
    text = "".join(ch for ch in text if ch.isprintable())
    if not text:
        return None
    if len(text) > _WORKER_TEXT_MAX:
        text = text[: _WORKER_TEXT_MAX - 1].rstrip() + "\u2026"
    return text


def _worker_failure_text(status: int) -> str:
    """What the dashboard says when the worker's answer has no reason it can show."""
    if status == 404:
        return "The worker does not know this request (HTTP 404); it may be older than the UI, so update it"
    return f"Worker request failed (HTTP {status}); the worker's log has the reason"


async def _proxy_to_worker(
    worker_id: int,
    method: str,
    path: str,
    *,
    json: dict[str, Any] | None = None,
    params: dict[str, str] | None = None,
    timeout: float = 30,
) -> dict[str, Any]:
    """Proxy one request to a worker's REST API and return its parsed JSON.

    Single home for the fetch-worker -> verified-URL+auth-header -> httpx call ->
    error-mapping sequence the deploy/command/logs paths all repeated. Dispatch uses
    the concrete httpx verbs (never client.request) so per-verb test mocks keep
    landing; the caller owns the timeout (deploy needs 60s, the rest 30s) and any
    query params (the logs line clamp, remove's delete_volumes).
    """
    worker = await database.get_worker(worker_id)
    url, headers = await _get_verified_worker_url(worker)

    try:
        async with httpx.AsyncClient(timeout=timeout) as client:
            verb = method.upper()
            if verb == "GET":
                resp = await client.get(f"{url}{path}", params=params, headers=headers)
            elif verb == "DELETE":
                resp = await client.delete(f"{url}{path}", params=params, headers=headers)
            else:
                resp = await client.post(f"{url}{path}", json=json, params=params, headers=headers)
            if resp.status_code >= 400:
                # NOT resp.text at warning level: withholding the raw body from the
                # caller (see _safe_worker_detail) is pointless if it goes straight
                # into the log instead. A deploy failure can echo env values, which
                # for several providers are live credentials.
                safe: dict[str, Any] | str | None = _safe_worker_detail(resp) or _worker_refusal_text(resp, json)
                logger.warning("worker proxy error (%s): %s", resp.status_code, safe or "unstructured error body")
                logger.debug("worker proxy error body (%s): %s", resp.status_code, resp.text)
                raise HTTPException(
                    status_code=resp.status_code,
                    detail=safe or _worker_failure_text(resp.status_code),
                )
            return resp.json()
    except (httpx.ConnectTimeout, httpx.PoolTimeout) as exc:
        # Both are TimeoutException subclasses, but the request never reached
        # the worker, so "it may still be working on it" would be wrong.
        logger.warning("worker proxy error: %s", exc)
        raise HTTPException(status_code=503, detail="Worker communication failed")
    except httpx.TimeoutException as exc:
        logger.warning("worker proxy timeout after %ss: %s", timeout, exc)
        raise HTTPException(
            status_code=504,
            detail=f"The worker did not answer within {timeout:g} s. It may still be working on it; check the worker.",
        )
    except httpx.HTTPError as exc:
        logger.warning("worker proxy error: %s", exc)
        raise HTTPException(status_code=503, detail="Worker communication failed")


async def _proxy_worker_command(
    worker_id: int, command: str, slug: str, *, params: dict[str, str] | None = None, instance: int = 1
) -> dict[str, Any]:
    """Forward a container command (restart/stop/start/remove) to a worker.

    ``instance`` picks one copy of a service that runs more than once; the first
    copy is addressed without it, so an older worker sees the request it always did.
    """
    if command == "remove":
        return await _proxy_to_worker(worker_id, "DELETE", f"/api/containers/{slug}", params=params)
    return await _proxy_to_worker(
        worker_id,
        "POST",
        f"/api/containers/{slug}/{command}",
        params={"instance": str(instance)} if instance > 1 else None,
    )


async def _worker_features(worker_id: int) -> set[str]:
    """What the worker said it can do in its last heartbeat."""
    worker = await database.get_worker(worker_id) or {}
    features = _safe_json(worker.get("system_info", "{}"), {}).get("features")
    return {str(f) for f in features} if isinstance(features, list) else set()


async def _worker_instances(worker_id: int, slug: str) -> set[int]:
    """The copies of ``slug`` the worker's last heartbeat listed."""
    worker = await database.get_worker(worker_id) or {}
    copies: set[int] = set()
    for entry in _safe_json(worker.get("containers", "[]"), []):
        if isinstance(entry, dict) and entry.get("slug") == slug:
            try:
                copies.add(max(1, int(entry.get("instance") or 1)))
            except (TypeError, ValueError):
                copies.add(1)
    return copies


async def _proxy_worker_deploy(worker_id: int, slug: str, spec: dict[str, Any]) -> dict[str, Any]:
    """Forward a deploy command with full spec to a worker.

    The worker stops the container it replaces with the catalog's stop timeout
    (300 s for storj) before it pulls and starts the new one, so the wait has to
    cover that stop. With a flat 60 s the dashboard reported a failed deploy
    while the worker was still shutting the old node down cleanly.
    """
    docker_conf = (catalog.get_service(slug) or {}).get("docker") or {}
    raw = docker_conf.get("stop_timeout")
    try:
        stop_timeout = max(0, int(raw)) if raw is not None else 30
    except (TypeError, ValueError):
        stop_timeout = 30
    # Which build THIS worker runs is decided here, at the edge, from the
    # architecture in its heartbeat. The deployment record stays on the
    # catalog's default image: it is one row for the whole fleet, and a
    # Raspberry Pi's tag must not be replayed onto an x86 box.
    # A custom image is the operator's choice for this worker and is sent as is.
    custom = spec.pop("custom_image", None)
    if not custom and isinstance(docker_conf.get("image_by_arch"), dict):
        worker = await database.get_worker(worker_id) or {}
        arch = _safe_json(worker.get("system_info", "{}"), {}).get("arch")
        image = _image_for_arch(docker_conf, arch)
        if image:
            spec = {**spec, "image": image}
    return await _proxy_to_worker(
        worker_id, "POST", f"/api/containers/{slug}/deploy", json=spec, timeout=60 + stop_timeout
    )


async def _proxy_worker_logs(worker_id: int, slug: str, lines: int = 50, instance: int = 1) -> dict[str, str]:
    """Forward a logs request to a worker."""
    params = {"lines": min(lines, 1000)}
    if instance > 1:
        params["instance"] = instance
    return await _proxy_to_worker(worker_id, "GET", f"/api/containers/{slug}/logs", params=params)


# ---------------------------------------------------------------------------
# API: Proxies and proxied copies (app/proxies.py, docs/proxies.md)
# ---------------------------------------------------------------------------


class ProxyCopyRequest(BaseModel):
    worker_id: int
    slug: str


async def _proxy_or_404(proxy_id: int) -> dict[str, Any]:
    proxy = await database.get_proxy(proxy_id)
    if not proxy:
        raise HTTPException(status_code=404, detail="No such proxy")
    return proxy


def _mask(text: Any, secrets_: list[str]) -> Any:
    if not isinstance(text, str):
        return text
    for value in sorted(secrets_, key=len, reverse=True):
        if value:
            text = text.replace(value, "***")
    return text


async def _proxy_worker_call(
    worker_id: int, method: str, path: str, definition: dict[str, Any], *, json: Any = None, timeout: float = 30
) -> dict[str, Any]:
    """_proxy_to_worker for the proxy routes, with every proxy secret masked
    out of whatever the worker answers."""
    try:
        return await _proxy_to_worker(worker_id, method, path, json=json, timeout=timeout)
    except HTTPException as exc:
        raise HTTPException(status_code=exc.status_code, detail=_mask(exc.detail, proxies.secret_values(definition)))


async def _require_proxy_feature(worker_id: int) -> None:
    if "proxy_tunnel" not in await _worker_features(worker_id):
        raise HTTPException(
            status_code=409,
            detail="This worker is too old to run copies through a proxy, or has no Docker socket. Update it first.",
        )


@app.get("/api/proxies")
async def api_list_proxies(request: Request, _auth: dict[str, Any] = Depends(_require_owner)) -> dict[str, Any]:
    """Every proxy, without its secrets, and running copies grouped by exit IP."""
    items = await database.list_proxies()
    copies = await database.list_proxied_copies()
    workers = await database.list_workers()
    for w in workers:
        _parse_worker_json(w)
    rotating = {p["id"] for p in items if (p.get("definition") or {}).get("rotating")}
    return {
        "proxies": [proxies.public_view(p, [c for c in copies if c["proxy_id"] == p["id"]]) for p in items],
        "exits": proxies.proxy_exits(workers, rotating),
    }


@app.post("/api/proxies")
async def api_create_proxy(request: Request, _auth: dict[str, Any] = Depends(_require_owner)) -> dict[str, Any]:
    try:
        body = await request.json()
        definition = proxies.clean_definition(body if isinstance(body, dict) else {})
    except proxies.ProxyInputError as exc:
        raise HTTPException(status_code=400, detail=str(exc)) from None
    except ValueError:
        raise HTTPException(status_code=400, detail="Send the proxy as a JSON object") from None
    proxy_id = await database.create_proxy(definition)
    if definition.get("reset_day"):
        # The first period starts now; with no start the first hourly sweep
        # would take it for a missed reset and zero what was counted so far.
        await database.update_proxy_state(proxy_id, proxies.reset_state({}, datetime.now(UTC).date()))
    return proxies.public_view(await _proxy_or_404(proxy_id))


@app.patch("/api/proxies/{proxy_id}")
async def api_update_proxy(
    request: Request, proxy_id: int, _auth: dict[str, Any] = Depends(_require_owner)
) -> dict[str, Any]:
    await _proxy_or_404(proxy_id)
    try:
        body = await request.json()
    except ValueError:
        raise HTTPException(status_code=400, detail="Send the proxy as a JSON object") from None
    async with _proxy_state_lock(proxy_id):
        # Read again under the lock, so a heartbeat or reset that ran while the
        # body was parsed is neither lost nor allowed to undo this change.
        current = await _proxy_or_404(proxy_id)
        try:
            definition = proxies.clean_definition(body if isinstance(body, dict) else {}, current["definition"])
        except proxies.ProxyInputError as exc:
            raise HTTPException(status_code=400, detail=str(exc)) from None
        await database.update_proxy_definition(proxy_id, definition)
        # A cap turned on starts a fresh period; one turned off or raised above
        # this period's use lifts the stop and starts the copies it stopped.
        state, release = proxies.cap_change(
            current["definition"], definition, current["state"], datetime.now(UTC).date()
        )
        if state != current["state"]:
            await database.update_proxy_state(proxy_id, state)
        current = {**current, "definition": definition}
    if release:
        await _release_capped_copies({**current, "state": state})
    return proxies.public_view(await _proxy_or_404(proxy_id), await database.list_proxied_copies(proxy_id))


@app.delete("/api/proxies/{proxy_id}")
async def api_delete_proxy(proxy_id: int, _auth: dict[str, Any] = Depends(_require_owner)) -> dict[str, str]:
    await _proxy_or_404(proxy_id)
    # Under the lock a copy being added records itself first, so it is either
    # seen here or finds the proxy gone.
    async with _proxy_state_lock(proxy_id):
        if await database.list_proxied_copies(proxy_id):
            raise HTTPException(status_code=409, detail="Remove the copies running through this proxy first")
        await database.delete_proxy(proxy_id)
    return {"status": "deleted"}


@app.post("/api/proxies/{proxy_id}/test")
async def api_test_proxy(
    proxy_id: int, worker_id: int | None = None, _auth: dict[str, Any] = Depends(_require_owner)
) -> dict[str, Any]:
    """Bring up a throwaway tunnel on a worker and report the exit IP and country it gets."""
    proxy = await _proxy_or_404(proxy_id)
    worker_id = await _resolve_worker_id(worker_id)
    await _require_proxy_feature(worker_id)
    definition = proxy["definition"]
    worker = await database.get_worker(worker_id) or {}
    box = str(worker.get("name") or f"worker {worker_id}")
    try:
        result = await _proxy_worker_call(
            worker_id,
            "POST",
            "/api/proxy-test",
            definition,
            json={"proxy": proxies.worker_proxy_spec(definition)},
            timeout=90,
        )
    except HTTPException as exc:
        # Kept on the proxy, so the reason outlives the toast that showed it.
        await _patch_proxy_state(
            proxy_id,
            {
                "last_test": {
                    "ok": False,
                    "at": datetime.now(UTC).isoformat(timespec="seconds"),
                    "worker": box,
                    "error": str(exc.detail)[:400],
                }
            },
        )
        raise
    now_iso = datetime.now(UTC).isoformat(timespec="seconds")
    exit_ip = egress.public_ip(result.get("exit_ip"))
    country = str(result.get("country") or "") or None
    await _patch_proxy_state(
        proxy_id,
        {
            "last_exit_ip": exit_ip,
            "last_country": country,
            "last_checked_at": now_iso,
            "last_test": {"ok": True, "at": now_iso, "worker": box, "error": ""},
        },
    )
    return {"exit_ip": exit_ip, "country": country, "ms": result.get("ms"), "worker": box}


async def _patch_proxy_state(proxy_id: int, changes: dict[str, Any]) -> None:
    """Set some keys of a proxy's state on a fresh read.

    A test runs for up to 90 s; writing back the state read before it would
    undo whatever the heartbeats counted meanwhile, a cap hit included.
    """
    async with _proxy_state_lock(proxy_id):
        proxy = await database.get_proxy(proxy_id)
        if proxy:
            await database.update_proxy_state(proxy_id, {**proxy["state"], **changes})


# One lock per proxy, per event loop, around every read-modify-write of
# proxies.state. Heartbeats from several workers, the hourly reset job, a
# manual reset, an edit and a proxy test all write that one JSON column; two
# that overlap across an await would otherwise lose one write (a reset undone,
# a heartbeat's bytes dropped). Keyed by loop because an asyncio.Lock binds to
# the loop it first waits on.
_proxy_state_locks: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[int, asyncio.Lock]] = (
    weakref.WeakKeyDictionary()
)


def _proxy_state_lock(proxy_id: int) -> asyncio.Lock:
    locks = _proxy_state_locks.setdefault(asyncio.get_running_loop(), {})
    return locks.setdefault(proxy_id, asyncio.Lock())


# One lock per copy, held by the owner's start or stop and by the cap's own starts
# from the mark check to the mark update, so a start the cap decided on can never
# land after the owner's stop.
_proxy_copy_locks: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[tuple[int, int, str], asyncio.Lock]] = (
    weakref.WeakKeyDictionary()
)


def _proxy_copy_lock(proxy_id: int, worker_id: int, slug: str) -> asyncio.Lock:
    locks = _proxy_copy_locks.setdefault(asyncio.get_running_loop(), {})
    return locks.setdefault((proxy_id, worker_id, slug), asyncio.Lock())


async def _copy_marked_by_cap(proxy_id: int, worker_id: int, slug: str) -> bool:
    return any(
        c["worker_id"] == worker_id and c["slug"] == slug and c.get("stopped_by_cap")
        for c in await database.list_proxied_copies(proxy_id)
    )


@app.post("/api/proxies/{proxy_id}/copies")
async def api_add_proxied_copy(
    proxy_id: int, body: ProxyCopyRequest, _auth: dict[str, Any] = Depends(_require_owner)
) -> dict[str, Any]:
    """Run a copy of a service already deployed on a worker through this proxy.

    The copy is built from that deployment's own record, so it carries the
    same image, credentials and limits, under its own device names.
    """
    proxy = await _proxy_or_404(proxy_id)
    definition, state = proxy["definition"], proxy["state"]
    if state.get("capped_at"):
        raise HTTPException(status_code=409, detail=state.get("cap_reason") or "This proxy reached its traffic cap")
    slug = body.slug
    svc = catalog.get_service(slug) or await _custom_service(slug, reveal=True)
    if not svc:
        raise HTTPException(status_code=404, detail=f"Service '{slug}' not found")
    if svc.get("status") in _UNDEPLOYABLE_STATUSES:
        raise HTTPException(status_code=409, detail=f"Service '{slug}' is no longer available for deployment")
    if why := proxies.refusal(svc):
        raise HTTPException(status_code=400, detail=why)
    worker_id = await _resolve_worker_id(body.worker_id)
    await _require_proxy_feature(worker_id)
    recorded = await database.get_deployment_spec(slug, worker_id=worker_id)
    if not recorded:
        raise HTTPException(
            status_code=409,
            detail=f"Deploy {svc.get('name') or slug} on this worker first; the proxied copy runs with its settings.",
        )
    docker_conf = svc.get("docker") or {}
    try:
        earner = proxies.earner_spec(recorded, docker_conf, proxy_id)
    except ValueError as exc:
        raise HTTPException(status_code=400, detail=f"{svc.get('name') or slug}: {exc}") from exc
    if not recorded.get("custom_image") and isinstance(docker_conf.get("image_by_arch"), dict):
        worker = await database.get_worker(worker_id) or {}
        arch_image = _image_for_arch(docker_conf, _safe_json(worker.get("system_info", "{}"), {}).get("arch"))
        if arch_image:
            earner["image"] = arch_image
    # The record goes in before the worker is asked, under the proxy's lock: a
    # deploy that times out may still finish on the worker, and a copy with no
    # record would be outside the cap and the delete guard.
    async with _proxy_state_lock(proxy_id):
        fresh = await database.get_proxy(proxy_id)
        if not fresh:
            raise HTTPException(status_code=404, detail="No such proxy")
        if fresh["state"].get("capped_at"):
            raise HTTPException(
                status_code=409, detail=fresh["state"].get("cap_reason") or "This proxy reached its traffic cap"
            )
        definition = fresh["definition"]
        had_record = any(
            c["worker_id"] == worker_id and c["slug"] == slug for c in await database.list_proxied_copies(proxy_id)
        )
        await database.save_proxied_copy(proxy_id, worker_id, slug)
    try:
        result = await _proxy_worker_call(
            worker_id,
            "POST",
            f"/api/proxied/{slug}/{proxy_id}/deploy",
            definition,
            json={"earner": earner, "proxy": proxies.worker_proxy_spec(definition)},
            timeout=180,
        )
    except HTTPException as exc:
        # A 504 says the worker may still be deploying: the record stays, so
        # the copy is listed and can be removed. Any other answer is final.
        if exc.status_code != 504 and not had_record:
            await database.delete_proxied_copy(proxy_id, worker_id, slug)
        raise
    exit_info = result.get("exit") if isinstance(result.get("exit"), dict) else {}
    exit_ip = egress.public_ip(exit_info.get("ip"))
    workers = await database.list_workers()
    for w in workers:
        _parse_worker_json(w)
    response: dict[str, Any] = {"status": "deployed", "exit_ip": exit_ip, "country": exit_info.get("country") or None}
    # Preflight after the fact: the exit IP is only known once the tunnel is up.
    others = proxies.same_exit_copies(workers, slug, exit_ip, itself=(worker_id, proxy_id))
    if others:
        limit = egress.devices_per_ip_limit(svc)
        response["warning"] = f"{svc.get('name') or slug} already leaves from {exit_ip} on {', '.join(others)}. " + (
            "Its provider allows one device per IP, so this copy earns nothing there."
            if limit == 1
            else "Copies on one IP share its bandwidth, and the provider may limit devices per IP."
        )
    # The copy is a new device: add it to the account as a deploy does, or show the step.
    if step := catalog.device_link_step(svc, earner["env"]):
        registered = await _register_device(slug, step["id"], earner["env"])
        step["registered"] = registered is True
        step["retrying"] = registered is None
        response["device_link"] = step
    return response


@app.post("/api/proxies/{proxy_id}/copies/{worker_id}/{slug}/{command}")
async def api_proxied_copy_command(
    proxy_id: int, worker_id: int, slug: str, command: str, _auth: dict[str, Any] = Depends(_require_owner)
) -> dict[str, Any]:
    if command not in ("start", "stop"):
        raise HTTPException(status_code=400, detail="The command is start or stop")
    proxy = await _proxy_or_404(proxy_id)
    if command == "start" and proxy["state"].get("capped_at"):
        raise HTTPException(status_code=409, detail=proxy["state"].get("cap_reason") or "This proxy reached its cap")
    async with _proxy_copy_lock(proxy_id, worker_id, slug):
        if command == "start":
            # Again under the lock: the cap may have stopped the copy while this waited.
            proxy = await _proxy_or_404(proxy_id)
            if proxy["state"].get("capped_at"):
                raise HTTPException(
                    status_code=409, detail=proxy["state"].get("cap_reason") or "This proxy reached its cap"
                )
        result = await _proxy_worker_call(
            worker_id,
            "POST",
            f"/api/proxied/{slug}/{proxy_id}/{command}",
            proxy["definition"],
            # The proxy as stored now, so an edit since the deploy reaches the copy.
            json={"proxy": proxies.worker_proxy_spec(proxy["definition"])} if command == "start" else None,
            timeout=90,
        )
        # A start or stop by hand is the owner's decision; a later reset must not undo it.
        await database.set_proxied_copy_capped(proxy_id, worker_id, slug, False)
    return result


@app.delete("/api/proxies/{proxy_id}/copies/{worker_id}/{slug}")
async def api_remove_proxied_copy(
    proxy_id: int, worker_id: int, slug: str, _auth: dict[str, Any] = Depends(_require_owner)
) -> dict[str, Any]:
    proxy = await _proxy_or_404(proxy_id)
    if not await database.get_worker(worker_id):
        # The box was deleted or enrolled again under a new id: there is no
        # worker to ask, and keeping the row would also block deleting the proxy.
        await database.delete_proxied_copy(proxy_id, worker_id, slug)
        return {
            "status": "removed",
            "containers": [],
            "note": "That worker no longer exists; only the record was removed.",
        }
    result = await _proxy_worker_call(worker_id, "DELETE", f"/api/proxied/{slug}/{proxy_id}", proxy["definition"])
    await database.delete_proxied_copy(proxy_id, worker_id, slug)
    return result


@app.post("/api/proxies/{proxy_id}/reset-usage")
async def api_reset_proxy_usage(proxy_id: int, _auth: dict[str, Any] = Depends(_require_owner)) -> dict[str, Any]:
    """Start a new traffic period by hand, and start the copies the cap stopped."""
    proxy = await _proxy_or_404(proxy_id)
    started = await _reset_proxy(proxy)
    return {"status": "reset", "started": started}


async def _reset_proxy(proxy: dict[str, Any]) -> list[str]:
    async with _proxy_state_lock(proxy["id"]):
        fresh = await database.get_proxy(proxy["id"]) or proxy
        state = proxies.reset_state(fresh["state"], datetime.now(UTC).date())
        await database.update_proxy_state(proxy["id"], state)
    return await _release_capped_copies({**fresh, "state": state})


async def _release_capped_copies(proxy: dict[str, Any]) -> list[str]:
    """The cap no longer holds: clear its alert and start the copies it stopped."""
    await database.clear_alerts(kind="proxy_cap", subject=str(proxy["id"]))
    _drop_proxy_cap_bell(proxy["id"])
    started: list[str] = []
    for copy in await database.list_proxied_copies(proxy["id"]):
        key = (proxy["id"], copy["worker_id"], copy["slug"])
        if not copy.get("stopped_by_cap") or key in _proxy_cap_starting:
            continue
        _proxy_cap_starting.add(key)
        try:
            async with _proxy_copy_lock(*key):
                # The owner may have started or stopped it since the list was read,
                # and a heartbeat may have capped the proxy again.
                async with _proxy_state_lock(proxy["id"]):
                    fresh = await database.get_proxy(proxy["id"])
                    marked = await _copy_marked_by_cap(*key)
                if not fresh or fresh["state"].get("capped_at"):
                    return started
                if not marked:
                    continue
                await _start_proxied_copy(fresh, copy["worker_id"], copy["slug"])
                # The cap may have come back while the start ran: then the mark
                # stays, so the next heartbeat stops it and the next reset starts it.
                async with _proxy_state_lock(proxy["id"]):
                    again = await database.get_proxy(proxy["id"])
                    if again and not again["state"].get("capped_at"):
                        await database.set_proxied_copy_capped(proxy["id"], copy["worker_id"], copy["slug"], False)
            started.append(f"{copy['slug']} on worker {copy['worker_id']}")
        except HTTPException as exc:
            logger.warning(
                "Could not restart %s through proxy %s after a reset (its box's next heartbeat tries again): %s",
                copy["slug"],
                proxy["id"],
                exc.detail,
            )
        finally:
            _proxy_cap_starting.discard(key)
    return started


async def _start_proxied_copy(proxy: dict[str, Any], worker_id: int, slug: str) -> dict[str, Any]:
    """Start one copy with the proxy's current settings."""
    return await _proxy_worker_call(
        worker_id,
        "POST",
        f"/api/proxied/{slug}/{proxy['id']}/start",
        proxy["definition"],
        json={"proxy": proxies.worker_proxy_spec(proxy["definition"])},
        timeout=90,
    )


async def _run_proxy_cap_resets() -> None:
    """Scheduler job: start a new period for every proxy whose reset day has come."""
    today = datetime.now(UTC).date()
    for proxy in await database.list_proxies():
        if proxies.reset_due(proxy["definition"], proxy["state"], today):
            await _reset_proxy(proxy)


# Copies a capped proxy could not stop yet, per proxy: (worker_id, slug) -> why.
# Every heartbeat that still reports one running tries again.
_proxy_cap_unstopped: dict[int, dict[tuple[int, str], str]] = {}
# Stops already on their way, so a slow worker is not sent a second one.
_proxy_cap_stopping: set[tuple[int, int, str]] = set()
# Starts of copies the cap stopped (by a reset or a heartbeat retry) on their way,
# so the two paths never send a second start for the same copy.
_proxy_cap_starting: set[tuple[int, int, str]] = set()


def _running(entry: dict[str, Any]) -> bool:
    return str(entry.get("status")) in ("running", "restarting") or str(entry.get("tunnel_status")) in (
        "running",
        "restarting",
    )


async def _account_proxy_traffic(worker_id: int, entries: list[Any]) -> None:
    """Add one heartbeat's tunnel traffic to each proxy, and hold its copies at the cap.

    Heartbeats arrive every 60 s, so the copies can run up to a minute past the
    threshold; the 90% default leaves room for that (docs/proxies.md). While a
    proxy stays capped, every heartbeat that still reports one of its copies
    running stops it again, so a box that was unreachable at the crossing, or
    a copy the UI has no record of, cannot keep spending the allowance.
    """
    by_proxy: dict[int, list[dict[str, Any]]] = {}
    for entry in entries:
        if isinstance(entry, dict) and isinstance(entry.get("proxy_id"), int):
            by_proxy.setdefault(entry["proxy_id"], []).append(entry)
    if not by_proxy:
        return
    # Only traffic of copies the UI recorded on this worker counts, so a worker
    # cannot spend, or cap, a proxy it runs nothing through.
    copies = [c for c in await database.list_proxied_copies() if c["worker_id"] == worker_id]
    recorded = {(c["proxy_id"], c["slug"]) for c in copies}
    held_by_cap = {(c["proxy_id"], c["slug"]) for c in copies if c.get("stopped_by_cap")}
    now = time.time()
    for proxy_id, items in by_proxy.items():
        counted = [e for e in items if (proxy_id, egress.container_slug(e)) in recorded]
        async with _proxy_state_lock(proxy_id):
            proxy = await database.get_proxy(proxy_id)
            if not proxy:
                continue
            state, crossed = proxies.account(proxy["state"], proxy["definition"], worker_id, counted, now)
            await database.update_proxy_state(proxy_id, state)
        if not state.get("capped_at"):
            # A reset whose start failed leaves the copy marked as the cap's and
            # down; while the proxy stays uncapped, every heartbeat tries again.
            retry = {
                slug
                for e in items
                if not _running(e)
                and (proxy_id, slug := egress.container_slug(e)) in held_by_cap
                and (proxy_id, worker_id, slug) not in _proxy_cap_starting
            }
            if retry:
                _proxy_cap_starting.update((proxy_id, worker_id, slug) for slug in retry)
                _spawn(_restart_capped_copies(proxy_id, worker_id, retry))
            continue
        targets = {(worker_id, egress.container_slug(e)) for e in items if _running(e)}
        if crossed:
            # Every recorded copy, on every worker, is asked to stop. Only those
            # that were running are marked as stopped by the cap (see
            # _stop_capped_proxy), so one the owner stopped stays stopped.
            targets |= {(c["worker_id"], c["slug"]) for c in await database.list_proxied_copies(proxy_id)}
        targets = {t for t in targets if t[1] and (proxy_id, *t) not in _proxy_cap_stopping}
        if targets or crossed:
            _proxy_cap_stopping.update((proxy_id, *t) for t in targets)
            _spawn(_stop_capped_proxy({**proxy, "state": state}, targets))


async def _restart_capped_copies(proxy_id: int, worker_id: int, slugs: set[str]) -> None:
    """Start copies the cap stopped and a reset could not start, unless the cap is back."""
    try:
        for slug in sorted(slugs):
            async with _proxy_copy_lock(proxy_id, worker_id, slug):
                async with _proxy_state_lock(proxy_id):
                    fresh = await database.get_proxy(proxy_id)
                    # The owner may have started or stopped it since the heartbeat; that
                    # clears the mark, and the owner's decision stands. The copy lock
                    # keeps that true until this start has landed.
                    marked = await _copy_marked_by_cap(proxy_id, worker_id, slug)
                if not fresh or fresh["state"].get("capped_at"):
                    return
                if not marked:
                    continue
                try:
                    await _start_proxied_copy(fresh, worker_id, slug)
                except HTTPException as exc:
                    logger.warning(
                        "Could not restart %s through proxy %s after a reset: %s", slug, proxy_id, exc.detail
                    )
                    continue
                # The cap may have come back while the start ran. Then the mark stays, so
                # the next heartbeat stops the copy again and the next reset starts it.
                async with _proxy_state_lock(proxy_id):
                    again = await database.get_proxy(proxy_id)
                    if again and not again["state"].get("capped_at"):
                        await database.set_proxied_copy_capped(proxy_id, worker_id, slug, False)
    finally:
        _proxy_cap_starting.difference_update((proxy_id, worker_id, slug) for slug in slugs)


async def _stop_capped_proxy(proxy: dict[str, Any], targets: set[tuple[int, str]]) -> None:
    pid = proxy["id"]
    unstopped = _proxy_cap_unstopped.setdefault(pid, {})
    try:
        for worker_id, slug in sorted(targets):
            # The owner's start or stop of this copy waits, so its mark is never overwritten.
            async with _proxy_copy_lock(pid, worker_id, slug):
                try:
                    answer = await _proxy_worker_call(
                        worker_id, "POST", f"/api/proxied/{slug}/{pid}/stop", proxy["definition"], timeout=60
                    )
                    unstopped.pop((worker_id, slug), None)
                    # An older worker does not say; it is taken as running, as before.
                    if isinstance(answer, dict) and answer.get("was_running") is False:
                        continue
                    # A reset or a cap change may have lifted the cap while the stop
                    # was on its way; then the copy goes back up instead.
                    async with _proxy_state_lock(pid):
                        fresh = await database.get_proxy(pid)
                        still_capped = bool(fresh and fresh["state"].get("capped_at"))
                        if still_capped:
                            await database.set_proxied_copy_capped(pid, worker_id, slug, True)
                    if not still_capped and fresh:
                        try:
                            await _start_proxied_copy(fresh, worker_id, slug)
                        except HTTPException as exc:
                            # Marked, so the box's next heartbeat starts it.
                            await database.set_proxied_copy_capped(pid, worker_id, slug, True)
                            logger.warning("Could not start %s again through proxy %s: %s", slug, pid, exc.detail)
                except HTTPException as exc:
                    if exc.status_code == 404 and any(
                        t in str(exc.detail) for t in ("No proxied copy", "Worker not found")
                    ):
                        # Already gone from that box, or the box itself is gone: nothing left to stop.
                        unstopped.pop((worker_id, slug), None)
                        continue
                    unstopped[(worker_id, slug)] = str(exc.detail)[:200]
                    logger.warning("Could not stop %s at proxy %s's cap: %s", slug, pid, exc.detail)
    finally:
        _proxy_cap_stopping.difference_update((pid, *t) for t in targets)
    message = await _proxy_cap_message(proxy)
    if alert_id := await database.record_alert("proxy_cap", str(pid), message):
        name = (proxy.get("definition") or {}).get("name") or f"proxy {pid}"
        _spawn(_push_alert("proxy_cap", str(pid), f"CashPilot: {name} reached its traffic cap", message, alert_id))
    _set_proxy_cap_bell(pid, (proxy.get("definition") or {}).get("name") or f"proxy {pid}", message)


async def _proxy_cap_message(proxy: dict[str, Any]) -> str:
    reason = (proxy.get("state") or {}).get("cap_reason") or "This proxy reached its traffic cap"
    unstopped = _proxy_cap_unstopped.get(proxy["id"]) or {}
    if not unstopped:
        return f"Stopped: {reason}."
    names: dict[int, str] = {}
    with contextlib.suppress(Exception):
        names = {w["id"]: w["name"] for w in await database.list_workers()}
    still = ", ".join(f"{slug} on {names.get(wid, f'worker {wid}')}" for wid, slug in sorted(unstopped))
    return f"{reason}. Could not stop yet, retrying on every heartbeat: {still}."


def _set_proxy_cap_bell(proxy_id: int, name: str, message: str) -> None:
    global _collector_alerts
    _collector_alerts = [
        *(a for a in _collector_alerts if not (a.get("kind") == "proxy_cap" and a.get("proxy_id") == proxy_id)),
        {"kind": "proxy_cap", "platform": name, "proxy_id": proxy_id, "error": message},
    ]


def _drop_proxy_cap_bell(proxy_id: int) -> None:
    global _collector_alerts
    _proxy_cap_unstopped.pop(proxy_id, None)
    _collector_alerts = [
        a for a in _collector_alerts if not (a.get("kind") == "proxy_cap" and a.get("proxy_id") == proxy_id)
    ]


async def _proxy_cap_alerts() -> list[dict[str, Any]]:
    """Bell entries for every proxy that is capped right now, from the proxies table.

    Derived like _offline_worker_alerts, so neither the hourly rebuild of the
    bell nor a restart can drop a cap that still holds.
    """
    try:
        items = await database.list_proxies()
    except Exception as exc:  # noqa: BLE001 - the bell must not take down collection
        logger.warning("Could not derive proxy cap alerts: %s", exc)
        return []
    out: list[dict[str, Any]] = []
    for proxy in items:
        if (proxy.get("state") or {}).get("capped_at"):
            name = (proxy.get("definition") or {}).get("name") or f"proxy {proxy['id']}"
            out.append(
                {
                    "kind": "proxy_cap",
                    "platform": name,
                    "proxy_id": proxy["id"],
                    "error": await _proxy_cap_message(proxy),
                }
            )
    return out


# ---------------------------------------------------------------------------
# API: Service management (new-style routes matching frontend)
# ---------------------------------------------------------------------------


@app.post("/api/services/{slug}/restart")
async def api_service_restart(
    request: Request, slug: str, worker_id: int | None = None, instance: int = 1
) -> dict[str, str]:
    return await _svc_restart(request, slug, worker_id, _instance_param(instance))


@app.post("/api/services/{slug}/stop")
async def api_service_stop(
    request: Request, slug: str, worker_id: int | None = None, instance: int = 1
) -> dict[str, str]:
    return await _svc_stop(request, slug, worker_id, _instance_param(instance))


@app.post("/api/services/{slug}/start")
async def api_service_start(
    request: Request, slug: str, worker_id: int | None = None, instance: int = 1
) -> dict[str, str]:
    _require_writer(request)
    worker_id = await _resolve_worker_id(worker_id)
    result = await _proxy_worker_command(worker_id, "start", slug, instance=_instance_param(instance))
    await database.record_health_event(slug, "start")
    metrics.record_container_lifecycle("start", slug)
    return result


@app.get("/api/services/{slug}/logs")
async def api_service_logs(
    request: Request, slug: str, lines: int = 50, worker_id: int | None = None, instance: int = 1
) -> dict[str, str]:
    _require_writer(request)
    worker_id = await _resolve_worker_id(worker_id)
    return await _proxy_worker_logs(worker_id, slug, lines, instance=_instance_param(instance))


@app.delete("/api/services/{slug}")
async def api_service_remove(
    request: Request,
    slug: str,
    worker_id: int | None = None,
    delete_volumes: bool = False,
    allow_delete_critical: bool = False,
) -> dict[str, Any]:
    return await _svc_remove(request, slug, worker_id, delete_volumes, allow_delete_critical)


# ---------------------------------------------------------------------------
# API: Compose export
# ---------------------------------------------------------------------------


@app.get("/api/compose/{slug}", response_class=PlainTextResponse)
async def api_compose_single(request: Request, slug: str, arch: str | None = None):
    """Export a docker-compose.yml for a single service."""
    _require_auth_api(request)
    svc = catalog.get_service(slug)
    if not svc:
        raise HTTPException(status_code=404, detail=f"Service '{slug}' not found")
    try:
        return compose_generator.generate_compose_single(slug, arch=_compose_arch(arch))
    except ValueError as exc:
        raise HTTPException(status_code=400, detail=str(exc))


class ComposeMultiRequest(BaseModel):
    slugs: list[str]
    # amd64, arm64 or arm (a raw machine name such as aarch64 is accepted too).
    # Absent means the catalog's default image, which is the amd64 build.
    arch: str | None = None


def _compose_arch(value: str | None) -> str | None:
    """Validate the export's target architecture; None means the default image."""
    if value is None or not str(value).strip():
        return None
    fam = arch.family(value)
    if fam is None:
        raise HTTPException(
            status_code=400,
            detail=f"Unknown architecture '{value}': use one of {', '.join(sorted(arch.FAMILIES))}",
        )
    return fam


@app.post("/api/compose", response_class=PlainTextResponse)
async def api_compose_multi(
    request: Request,
    body: ComposeMultiRequest,
    _auth: dict[str, Any] = Depends(_require_auth_api),
):
    """Export a docker-compose.yml for multiple services."""
    try:
        return compose_generator.generate_compose_multi(body.slugs, arch=_compose_arch(body.arch))
    except ValueError as exc:
        raise HTTPException(status_code=400, detail=str(exc))


@app.get("/api/compose", response_class=PlainTextResponse)
async def api_compose_all(request: Request, arch: str | None = None):
    """Export a docker-compose.yml for ALL services with Docker images."""
    _require_auth_api(request)
    try:
        return compose_generator.generate_compose_all(arch=_compose_arch(arch))
    except ValueError as exc:
        raise HTTPException(status_code=400, detail=str(exc))


# ---------------------------------------------------------------------------
# API: Earnings
# ---------------------------------------------------------------------------


@app.get("/api/earnings")
async def api_earnings(request: Request) -> list[dict[str, Any]]:
    _require_auth_api(request)
    return await database.get_earnings_summary()


@app.get("/api/earnings/summary")
async def api_earnings_summary(request: Request) -> dict[str, Any]:
    """Aggregated earnings stats for the dashboard."""
    _require_reader(request)
    summary = await database.get_earnings_dashboard_summary()

    # Load config for signup bonus offsets
    all_config = await database.get_config()
    if not isinstance(all_config, dict):
        all_config = {}

    # Include non-USD balances converted to USD in the total.
    # Compute total_adjusted as the sum of clamped per-service adjusted
    # balances (converted to USD) so it always matches the breakdown view.
    all_earnings = await database.get_earnings_summary()
    total_bonus_usd = 0.0
    total_adjusted = 0.0
    # CashPilot-oj4: the count was computed deep in database.py and only ever
    # logged. A total that silently omits holdings is indistinguishable from a
    # correct one, so the number of omissions has to reach the response.
    unpriced_platforms: list[str] = []
    for e in all_earnings:
        slug = e.get("platform", "")
        balance = float(e["balance"])
        currency = e["currency"]

        bonus = 0.0
        with contextlib.suppress(ValueError, TypeError):
            bonus = float(all_config.get(f"{slug}_signup_bonus", "0") or "0")
        adjusted = max(0.0, balance - bonus)

        if currency != "USD":
            # Fall back to the rate the reading was RECORDED at.
            #
            # to_usd consults only the live caches, so a crypto whose rate
            # lookup is merely stale was dropped from the headline total
            # entirely — silently, with nothing on screen saying the figure was
            # incomplete. upsert_earnings preserves fx_rate_usd on every row
            # precisely so a later reader is not left guessing.
            #
            # A stored rate is imperfect (it is the rate at collection time,
            # not now) but it is enormously better than omitting the holding,
            # and the count below says how many rows needed it.
            stored_rate = database._usd_rate(currency, e.get("fx_rate_usd"))
            usd_val = _to_usd_with_stored(balance, currency, stored_rate)
            if usd_val is not None:
                summary["total"] = round(summary["total"] + usd_val, 2)
            elif currency.upper() not in exchange_rates.UNPRICED_BY_DESIGN:
                # Points are not money, so leaving them out does not make the
                # Total partial; the card shows them in their own units.
                unpriced_platforms.append(slug)
            adj_usd = _to_usd_with_stored(adjusted, currency, stored_rate)
            if adj_usd is not None:
                total_adjusted += adj_usd
            bonus_usd = _to_usd_with_stored(bonus, currency, stored_rate) if bonus > 0 else 0.0
            if bonus_usd is not None:
                total_bonus_usd += bonus_usd
        else:
            total_adjusted += adjusted
            total_bonus_usd += bonus

    # Count active (running) services from worker data.
    #
    # None, not 0, when the count could not be taken. _get_all_worker_containers
    # opens SQLite, so a locked or busy database — or a JSON-decode failure on a
    # worker row — lands here while containers are in fact running, and "0"
    # reads as "nothing is running". Logged at WARNING rather than DEBUG: DEBUG
    # is off in production, so the only two places that could have said anything
    # both said nothing (CashPilot-45k).
    active: int | None = None
    try:
        worker_containers = await _get_all_worker_containers()
        active = sum(1 for s in worker_containers if s.get("status") == "running")
    except Exception as exc:
        logger.warning("Could not count active services, reporting it as unknown: %s", exc)
    summary["active_services"] = active
    # A total that silently omits holdings is indistinguishable from a correct
    # one. The count was already being computed in database.py and only logged;
    # it now reaches the caller so the card can say the figure is partial.
    # Whether anything has ever been read at all. Without this the payload is
    # identical on a fresh install and on an install whose collection has
    # silently stopped — an expired cookie, deleted credentials, a wedged
    # scheduler — and the dashboard states "$0.00" as a measurement in both.
    # A new user's very first view of CashPilot asserted three zero balances in
    # the same typeface those cards will later carry real money in.
    summary["has_readings"] = bool(all_earnings)
    summary["unpriced_platforms"] = sorted(set(unpriced_platforms))
    summary["total_bonus"] = round(total_bonus_usd, 2)
    summary["total_adjusted"] = round(total_adjusted, 2)
    # The balance total drops every time the user cashes out, so on its own it
    # says how much is waiting, never how much the services have made. Confirmed
    # payouts, valued at the rate they landed at, are the rest of that answer.
    # None, not 0, when the payouts could not be read: 0 would claim nothing was
    # ever cashed out. The base is the figure the balance card shows, so promo
    # credit the card subtracts is not counted back in here.
    paid_out: float | None = None
    unpriced_payouts = 0
    try:
        node_totals = await _node_totals()
        # A provider read node by node has its settlements in the node totals
        # below, which include any payout confirmed before (from the old
        # balance-drop question), so those are left out rather than counted twice.
        paid_out = sum(
            total
            for platform, total in (await database.get_confirmed_payout_totals()).items()
            if platform not in node_totals
        )
        unpriced_payouts = await database.count_unpriced_confirmed_payouts(exclude=set(node_totals))
        # What nodes settled on their own (Mysterium, issue #488) is a payout
        # too: it left the balance and reached the wallet, less the provider's
        # fee. Priced at today's rate, the only one there is for a total that
        # no single day owns. With the fee unknown it is left out and counted
        # like an unpriced payout, so the card says the total is low.
        for node_total in node_totals.values():
            if node_total["settled_gross"] <= 0:
                continue
            settled = node_total["settled"]
            settled_usd = None if settled is None else exchange_rates.to_usd(settled, node_total["currency"])
            if settled_usd is None:
                unpriced_payouts += 1
            else:
                paid_out += settled_usd
    except Exception as exc:  # noqa: BLE001 - the balance cards must still render
        logger.warning("Could not total confirmed payouts, reporting them as unknown: %s", exc)
    balance_shown = summary["total_adjusted"] if summary["total_bonus"] > 0 else summary["total"]
    summary["paid_out"] = None if paid_out is None else round(paid_out, 2)
    summary["earned_all_time"] = None if paid_out is None else round(balance_shown + paid_out, 2)
    # Payouts in a token with no recorded rate are left out of paid_out, so the
    # all-time figure is an understatement by that many; the card says so.
    summary["unpriced_payouts"] = unpriced_payouts
    return summary


@app.get("/api/earnings/daily")
async def api_earnings_daily(request: Request, days: int = 7) -> list[dict[str, Any]]:
    """Daily earnings for charting."""
    _require_auth_api(request)
    if days < 1 or days > 365:
        raise HTTPException(status_code=400, detail="days must be between 1 and 365")
    return await database.get_daily_earnings(days)


@app.get("/api/earnings/breakdown")
async def api_earnings_breakdown(request: Request) -> list[dict[str, Any]]:
    """Per-service earnings breakdown with cashout eligibility."""
    _require_reader(request)
    rows = await database.get_earnings_per_service()

    # Load config for per-service signup bonus offsets
    all_config = await database.get_config()
    if not isinstance(all_config, dict):
        all_config = {}

    result = []
    for row in rows:
        slug = row["platform"]
        svc = catalog.get_service(slug)
        cashout = (svc.get("cashout", {}) if svc else {}) or {}
        # Three-valued, matching payouts.min_payout: a positive number is a
        # real minimum, 0 is a documented "no minimum", and None means nobody
        # published one. float(..., 0) collapsed the last two, so 21 services
        # with no published minimum reported the user as eligible to cash out
        # any balance above zero.
        raw_min = cashout.get("min_amount")
        try:
            min_amount = float(raw_min) if raw_min is not None else None
        except (TypeError, ValueError):
            min_amount = None
        balance = float(row["balance"])
        # The minimum in the same unit as this balance. The catalog declares it
        # in whatever the provider cashes out in, which for Storj (USD balance,
        # STORJ minimum) and anyone-protocol is not what the collector reports —
        # so `balance >= min_amount` was comparing two different currencies at
        # 1:1, and told the user they were eligible when they were not.
        comparable_min = payouts.min_payout_in(svc, row.get("currency"))
        prev_balance = float(row.get("prev_balance", 0))
        delta = balance - prev_balance

        # Signup bonus offset (stored in config as {slug}_signup_bonus)
        signup_bonus = 0.0
        with contextlib.suppress(ValueError, TypeError):
            signup_bonus = float(all_config.get(f"{slug}_signup_bonus", "0") or "0")
        balance_adjusted = round(max(0.0, balance - signup_bonus), 4)

        entry = {
            "platform": slug,
            "name": svc["name"] if svc else slug,
            "balance": round(balance, 4),
            "balance_adjusted": balance_adjusted,
            "signup_bonus": round(signup_bonus, 4),
            "currency": row["currency"],
            "last_updated": row["date"],
            "delta": round(delta, 4),
            "cashout": {
                # None, not True, when the minimum is unknown. Claiming
                # eligibility we cannot establish sends the user to a withdrawal
                # page that will refuse them.
                # False and None mean different things here, and the existing
                # tests were right to insist on the distinction:
                #   no cashout section  -> False. There is no withdrawal route
                #                          at all; that is a real answer.
                #   route but no known  -> None. We cannot say, and claiming
                #   minimum                eligibility would send the user to a
                #                          page that refuses them.
                "eligible": (
                    False
                    if not cashout
                    else (None if comparable_min is None else (balance > 0 and balance >= comparable_min))
                ),
                "min_amount": min_amount,
                # The threshold in the same unit as the balance above, or None
                # when the two cannot be reconciled. Eligibility is decided on
                # this one; min_amount stays as the catalog declares it so the
                # provider's own wording is still available to the UI.
                "min_amount_comparable": comparable_min,
                "min_amount_currency": payouts.min_payout_currency(svc),
                "method": cashout.get("method", "redirect"),
                "dashboard_url": cashout.get("dashboard_url", ""),
                "notes": cashout.get("notes", ""),
            },
        }
        result.append(entry)
    return result


@app.get("/api/earnings/history")
async def api_earnings_history(request: Request, period: str = "week") -> list[dict[str, Any]]:
    _require_auth_api(request)
    if period not in ("week", "month", "year", "all"):
        raise HTTPException(status_code=400, detail="period must be week, month, year, or all")
    return await database.get_earnings_history(period)


@app.get("/api/health/scores")
async def api_health_scores(request: Request, days: int = 7) -> list[dict[str, Any]]:
    """Health scores for all services."""
    _require_reader(request)
    if days < 1 or days > 90:
        raise HTTPException(status_code=400, detail="days must be between 1 and 90")
    scores = await database.get_health_scores(days)
    # Enrich with service names
    for s in scores:
        svc = catalog.get_service(s["slug"])
        s["name"] = svc["name"] if svc else s["slug"]
    return scores


@app.post("/api/collect")
async def api_collect(request: Request) -> dict[str, str]:
    _require_writer(request)
    _spawn(_run_collection())
    return {"status": "collection_started"}


_MAX_ALERT_ERROR_LEN = 200


@app.get("/api/earnings/flatlines")
async def api_earnings_flatlines(request: Request) -> list[dict[str, Any]]:
    """Services that are running but whose balance has stopped moving."""
    _require_auth_api(request)
    return await database.get_flatlined_services(quiet_days=_quiet_days_by_platform())


@app.get("/api/credentials/health")
async def api_credential_health(request: Request) -> list[dict[str, Any]]:
    """Report how old each stored credential is and when it is expected to die.

    Never returns credential VALUES - only which key it is, how old, and what
    that means. The point is that "this will stop working tonight" is visible
    BEFORE it happens, rather than the user discovering it because earnings
    quietly stopped being recorded.
    """
    _require_auth_api(request)

    from app.collectors import _COLLECTOR_ARGS, credential_lifetime, durable_alternative
    from app.credential_expiry import jwt_times

    updated = await database.get_config_updated_at()
    # The values themselves, decrypted, so a JWT's own expiry can be read. They
    # stay inside this function: the report carries dates, never the value.
    try:
        values = await database.get_config()
    except Exception:  # noqa: BLE001 - a value-less report is still a report
        values = {}
    if not isinstance(values, dict):
        values = {}
    now = datetime.now(UTC)
    report: list[dict[str, Any]] = []
    # Services whose latest collection was REJECTED by the provider. That is
    # evidence about the credential, where its age is only a guess, so it wins
    # over both the lifetime table and a JWT's own claim.
    rejected = {
        a.get("platform")
        for a in _collector_alerts
        if a.get("kind", "collector") == "collector" and a.get("category") == "auth"
    }

    for slug, args in _COLLECTOR_ARGS.items():
        missing_durable = [field for field in durable_alternative(slug) if f"{slug}_{field}" not in updated]
        for arg in args:
            field = arg.lstrip("?")
            key = f"{slug}_{field}"
            stamp = updated.get(key)
            if not stamp:
                continue  # not configured; nothing to report an age for

            meta = credential_lifetime(slug, field) or {}
            hours_total = meta.get("hours")
            try:
                age_hours = (now - datetime.fromisoformat(stamp).replace(tzinfo=UTC)).total_seconds() / 3600
            except ValueError:
                continue

            if hours_total is None:
                status = "no_known_expiry"
            elif age_hours >= hours_total:
                status = "likely_expired"
            elif age_hours >= hours_total * 0.75:
                status = "expiring_soon"
            else:
                status = "fresh"

            # A JWT says when it dies. That beats both the paste date and the
            # per-service guess: a token pasted half-spent, or one the provider
            # now issues for 7 days instead of a year, is judged on its own
            # claim. The 25% window mirrors the age-based one; 48 h is the floor
            # so a short token still gets a warning a human can act on.
            issued_at, expires_at = jwt_times(str(values.get(key) or ""))
            if expires_at is not None:
                remaining_hours = (expires_at - now).total_seconds() / 3600
                span_hours = (
                    (expires_at - issued_at).total_seconds() / 3600
                    if issued_at is not None and expires_at > issued_at
                    else None
                )
                warn_hours = max(48.0, span_hours * 0.25) if span_hours else 48.0
                if remaining_hours <= 0:
                    status = "likely_expired"
                elif remaining_hours <= warn_hours:
                    status = "expiring_soon"
                else:
                    status = "fresh"
                if span_hours:
                    hours_total = round(span_hours)

            # A short-lived credential backed by its durable alternative is
            # re-established on every run (Bytelixir's 2 h session cookie by the
            # year-long remember-me cookie), so its own age says nothing.
            # Its lifetime and its "dies this afternoon" reason would then
            # contradict the calm status, so the row explains the cover instead.
            # Assumes every durable field of a service re-creates its
            # short-lived ones; true for Bytelixir, check it for the next one.
            why = meta.get("why")
            covered = bool(durable_alternative(slug)) and not missing_durable and not meta.get("durable")
            if covered and expires_at is None:
                status = "no_known_expiry"
                hours_total = None
                why = (
                    "Short-lived on its own, but re-created on every run from the saved "
                    + " and ".join(durable_alternative(slug))
                    + ", so its age does not matter."
                )

            if slug in rejected:
                status = "likely_expired"
                # The age-based reason (or "its age does not matter") would
                # contradict the status; the refusal is the reason now.
                why = (
                    "The provider rejected the saved credentials on the last run, "
                    "so they need replacing whatever their age."
                )

            entry: dict[str, Any] = {
                "service": slug,
                "field": field,
                "age_hours": round(age_hours, 1),
                "expected_lifetime_hours": hours_total,
                "status": status,
            }
            if expires_at is not None:
                entry["expires_at"] = expires_at.isoformat()
            if why:
                entry["why"] = why
            # Only nag about a durable alternative when the short-lived credential
            # is the one actually at risk.
            if missing_durable and hours_total is not None and not meta.get("durable"):
                entry["durable_alternative_missing"] = missing_durable
            report.append(entry)

    return report


@app.get("/api/services/{slug}/preflight")
async def api_service_preflight(
    request: Request, slug: str, worker_id: int | None = None, planned: str | None = None
) -> dict[str, Any]:
    """What this service will realistically do for THIS user, before deploying.

    Never blocks a deploy: the goal is informed consent, not a nanny.
    """
    _require_auth_api(request)
    service = catalog.get_service(slug)
    if not service:
        raise HTTPException(status_code=404, detail=f"Unknown service '{slug}'")

    # What is already running on the SAME machine is what makes a per-IP limit
    # checkable at all, so scope to that worker when we know which one.
    if worker_id is None:
        deployed = await database.get_deployments()
        return preflight.assess(service, already_deployed_slugs={d["slug"] for d in deployed})

    # Only ONLINE workers count as peers. A worker that is merely switched off
    # keeps its row forever once enrolled (the purge spares enrolled rows), and
    # its last heartbeat still lists every container as running — so a retired
    # machine would fabricate a conflict against a live one. This module
    # promises the opposite failure direction: a missed conflict, never an
    # invented one. The worker being deployed to is looked up separately so a
    # freshly-restarted one can still be assessed.
    all_workers = [_decoded_worker(w) for w in await database.list_workers()]
    workers = [w for w in all_workers if w.get("status") == "online"]
    worker = next((w for w in all_workers if w.get("id") == worker_id), None)
    if worker is None:
        raise HTTPException(status_code=404, detail=f"Unknown worker {worker_id}")

    # The other workers receiving this service in the SAME wizard action. The
    # cross-machine check otherwise only sees what is already RUNNING, so
    # ticking two boxes behind one connection produced no warning: neither
    # worker had the service yet, so neither counted against the other. The
    # warning arrived on the next deploy, after the account was already at risk
    # (CashPilot-3tr).
    planned_ids: set[int] = set()
    for part in (planned or "").split(","):
        part = part.strip()
        if part.isdigit():
            planned_ids.add(int(part))
    planned_ids.discard(worker_id)

    return preflight.assess(
        service,
        already_deployed_slugs=egress.running_slugs(worker),
        system_info=worker.get("system_info") or {},
        # The cross-machine half: providers cap per IP, so a sibling worker
        # behind the same public address is the case a single-host tool cannot
        # see at all.
        worker=worker,
        fleet_workers=workers,
        also_deploying_to=planned_ids,
    )


@app.get("/api/fleet/egress-groups")
async def api_fleet_egress_groups(request: Request) -> dict[str, Any]:
    """The fleet grouped by the public address providers actually see.

    The normal fleet view is by machine, which is the wrong unit: two machines
    in one house are two rows here and one customer to every provider.
    """
    _require_auth_api(request)
    workers = [_decoded_worker(w) for w in await database.list_workers()]
    groups = egress.group_by_egress(workers)
    return {
        "groups": [
            {
                "egress_ip": g["egress_ip"],
                "known": g["known"],
                "network_type": g["network_type"],
                "shared": g["shared"],
                "worker_count": g["worker_count"],
                "workers": [
                    {"id": w.get("id"), "name": w.get("name"), "client_id": w.get("client_id")} for w in g["workers"]
                ],
            }
            for g in groups
        ],
        "shared_groups": sum(1 for g in groups if g["shared"]),
        # Reported separately and never folded into the groups above: these are
        # machines whose exit we could not determine, not machines we checked.
        "undetermined": sum(g["worker_count"] for g in groups if not g["known"]),
    }


@app.get("/api/earnings/net")
async def api_earnings_net(request: Request, days: int = 30) -> dict[str, Any]:
    """Gross, estimated electricity cost, and net per service.

    Reports net ALONGSIDE gross, never instead of it, and every cost carries
    whether it was estimated or measured. With no tariff configured it reports
    gross and says the cost is unknown, rather than charging zero and quietly
    presenting gross as if it were profit.
    """
    _require_auth_api(request)

    cfg = await database.get_config()
    try:
        price = _tariff_price(cfg)
    except (TypeError, ValueError):
        price = 0.0
    currency = str(cfg.get("power_currency") or "EUR")
    try:
        host_tdp = float(cfg.get("power_host_tdp_watts") or power.DEFAULT_HOST_TDP_WATTS)
    except (TypeError, ValueError):
        host_tdp = power.DEFAULT_HOST_TDP_WATTS

    # Whether the power side of this endpoint has any input at all. An empty
    # `statuses` from a FAILURE is not the same fact as an empty one from a
    # fleet with nothing running, and the difference decides whether a net
    # figure exists (CashPilot-c6u).
    watts_known = True
    try:
        statuses = await _get_all_worker_containers()
    except Exception as exc:
        # A worker status problem must not take out the earnings figures, which
        # come from the database and are still perfectly reportable.
        logger.warning("Worker status unavailable for the power estimate: %s", exc)
        statuses = []
        watts_known = False
    # Only RUNNING containers. A stopped one draws nothing, and counting it
    # inflates a worker's container count, which shrinks every running service's
    # share of that host's idle floor and understates the fleet's real cost.
    # egress.container_slug, NOT c["service"].
    #
    # _get_all_worker_containers emits "slug"; this filtered on "service" and so
    # matched NOTHING in production, every time. The consequence was not a
    # missing panel — it was that every service was charged 0 W, so cost came
    # out 0.00 and net was reported EQUAL TO GROSS with cost_known: true. The
    # endpoint's own docstring promises it never presents gross as profit, and
    # that is exactly what it did.
    #
    # This is the second time this key has bitten the project. container_slug()
    # exists because of the first time and accepts both spellings; its docstring
    # says code reading "service" matched nothing in production while its tests
    # passed, which is precisely what happened here again.
    running = [c for c in statuses if egress.container_slug(c) and str(c.get("status", "")).lower() == "running"]

    # Group by WORKER and keep it that way through the watt calculation. Each
    # host pays its own idle draw, so collapsing a multi-host fleet into one
    # count charges a single idle floor for the whole estate. Keeping the group
    # is also the only way power.is_metered can be applied: a VPS bill does not
    # move with CPU, and billing it like a home server invents a cost.
    by_worker: dict[Any, list[dict[str, Any]]] = {}
    for c in running:
        by_worker.setdefault(c.get("_worker_id"), []).append(c)

    worker_meta = {w.get("id"): w for w in await database.list_workers()}

    hours = max(1, int(days)) * 24.0
    # Earned OVER THE WINDOW, not the latest balance: subtracting a window's
    # electricity from a running total would be meaningless arithmetic.
    earned = await database.get_earned_by_platform(max(1, int(days)))

    watts_by_service: dict[str, float] = {}
    unmetered_services: set[str] = set()
    for wid, containers in by_worker.items():
        meta = worker_meta.get(wid) or {}
        info = _safe_json(meta.get("system_info", "{}"), {})
        metered = power.is_metered(info)
        try:
            tdp = float(info.get("host_tdp_watts") or host_tdp)
        except (TypeError, ValueError):
            tdp = host_tdp
        count = max(1, len(containers))
        for c in containers:
            svc = egress.container_slug(c)
            if not metered:
                # No marginal power cost to the user on this host, so charge
                # nothing rather than computing watts and multiplying by zero.
                unmetered_services.add(svc)
                continue
            # A container whose CPU could not be read contributes no estimate
            # rather than an estimate built from a fabricated 0%. `or 0.0` below
            # would silently turn "unmeasurable" into "idle", which is how a
            # busy service ends up costing nothing on the running-costs card.
            if c.get("cpu_percent") is None:
                continue
            watts_by_service[svc] = watts_by_service.get(svc, 0.0) + power.estimate_watts(
                float(c.get("cpu_percent") or 0.0), host_tdp_watts=tdp, container_count=count
            )

    rows = []
    for platform, gross in earned.items():
        watts = watts_by_service.get(platform, 0.0)
        rows.append(
            {
                "platform": platform,
                "gross": float(gross),
                "watts": watts,
                "hours": hours,
                "cost_quality": power.ESTIMATED,
                # Whether a PER-SERVICE cost means anything, as opposed to
                # whether the machine costs anything. A bandwidth container adds
                # roughly 1-3 W to a host that is already on, which is below
                # what a consumer smart plug can resolve, so the share-out is
                # arithmetic rather than measurement.
                #
                # The cost itself is still reported and still counted in the
                # fleet total — the machine really does draw that power, and
                # dropping it would understate what the fleet costs. What is
                # unreliable is which service to blame, so that is what gets
                # flagged. machine_economics has held this rule since it was
                # written and nothing had ever asked it.
                "cost_attributable": machine_economics.per_service_is_meaningful(watts),
            }
        )

    # Convert the TARIFF into USD, rather than the gross into the tariff currency.
    #
    # gross comes from get_earned_by_platform, which is USD by contract, while
    # the tariff is whatever power_currency says. Subtracting one from the other
    # produced a number in neither, labelled with the tariff currency — at
    # ~1.08 USD/EUR an 8% error, in the direction that flatters the result, on
    # the figure that decides whether a machine is worth running.
    #
    # Converting the PRICE keeps this endpoint canonical USD like every other
    # money figure in the API, and the frontend's display-currency layer renders
    # it in whatever the user reads in. Converting the gross the other way would
    # have made this one endpoint the exception.
    price_usd = price
    fx_ok = True
    if price and currency != "USD":
        converted = exchange_rates.to_usd(price, currency)
        if converted is None:
            fx_ok = False
        else:
            price_usd = converted

    # No rate means the tariff cannot be expressed in the same unit as the
    # earnings, so there is no honest net. Reporting gross alone is what this
    # module already does when no tariff is set at all: "a zero cost would
    # render gross as net and quietly overstate earnings."
    #
    # Unknown WATTS suppress the net for the same reason unknown FX does, and
    # through the same mechanism: a zero price makes summarise report cost and
    # net as None instead of as numbers. Without this, a failed worker lookup
    # charged every service 0 W, cost summed to 0.00, and the endpoint returned
    # total_net == total_gross with cost_known TRUE -- gross presented as profit,
    # which is the one thing this endpoint's docstring promises it never does.
    # cost_known came from `price_per_kwh > 0` alone and never asked whether any
    # watts had actually been measured (CashPilot-c6u).
    result = power.summarise(rows, price_per_kwh=(price_usd if (fx_ok and watts_known) else 0.0), currency="USD")
    if not watts_known:
        result["cost_known"] = False
        result["watts_unavailable"] = True
        result["cost_unavailable_reason"] = (
            "CashPilot could not reach its workers, so it does not know what any service "
            "is drawing right now. Earnings below are real; the running cost and the net "
            "figure are unknown for this window, not zero."
        )
    if not fx_ok:
        result["cost_known"] = False
        result["fx_unavailable"] = True
        result["tariff_currency"] = currency
        result["cost_unavailable_reason"] = (
            f"Your tariff is in {currency} and earnings are recorded in USD. No "
            f"{currency}-to-USD rate is available right now, so a net figure would be "
            "two different currencies subtracted from each other."
        )
    result["window_days"] = max(1, int(days))
    result["host_tdp_watts"] = host_tdp
    return result


#: Hostname shape a resolver should ever be asked about. advertised_address is
#: WORKER-SUPPLIED text: without this gate a rogue worker could drive the hub's
#: resolver with arbitrary strings (and an IDNA-invalid label does not even
#: raise the OSError family — see _resolve_advertised_host).
_HOSTNAME_RE = re.compile(r"^[A-Za-z0-9]([A-Za-z0-9.-]{0,251}[A-Za-z0-9])?$")


async def _advertised_address_mismatch(matches: list[dict[str, Any]], worker_id: int | None = None) -> str | None:
    """Is the network dialling an address this machine no longer has?

    Compares the advertised_address a worker reported (the ONE env var the
    service's catalog entry declares, see advertised_address_env in
    services/_schema.yml) against that worker's detected egress IP. This is
    the check that catches a stale advertised IP the day the ISP re-provisions
    it, instead of days later via the provider's offline emails.

    Every failure to gather either side returns None — no claim — because a
    wrong "your address is stale" sends the operator to fix DNS that is fine.
    Only a definitive NXDOMAIN counts as a resolution verdict; a timeout or a
    transient resolver error is indistinguishable from OUR network having a
    bad moment, which says nothing about the record.
    """
    # Judge only what is actually RUNNING, and only on the node the caller
    # asked about: an exited container elsewhere in the fleet carries the env
    # of its LAST run, and judging that produced a finding about a machine the
    # page was not even showing.
    candidates = [c for c in matches if str(c.get("status", "")).lower() == "running"]
    if worker_id is not None:
        candidates = [c for c in candidates if c.get("_worker_id") == worker_id]
    reported = next((c for c in candidates if c.get("advertised_address") and c.get("_worker_id") is not None), None)
    if reported is None:
        return None
    row = await database.get_worker(reported["_worker_id"])
    if row is None:
        return None
    egress_ip = egress.egress_of(_decoded_worker(row))

    host = egress.advertised_host(reported["advertised_address"])
    resolved: set[str] | None = None
    if host and egress_ip:
        try:
            ipaddress.ip_address(host)
        except ValueError:
            if not _HOSTNAME_RE.fullmatch(host):
                return None
            resolved = await _resolve_advertised_host(host)
    reason = egress.advertised_address_verdict(reported["advertised_address"], egress_ip, resolved)
    # Name the node: on a fleet the bare reason says "this machine" with no
    # way to tell WHICH machine, which is a finding nobody can act on.
    if reason and (node := reported.get("_node")):
        return f"On {node}: {reason}"
    return reason


#: host -> (monotonic stamp, result). One entry per advertised hostname (one
#: service per fleet declares one today), so this stays tiny; the TTL exists so
#: a blackholed resolver costs one executor thread per window, not one per
#: request — wait_for abandons the await but the thread blocks on for the full
#: OS resolver timeout.
_RESOLVE_CACHE: dict[str, tuple[float, set[str] | None]] = {}
_RESOLVE_CACHE_TTL = 60.0
#: The keys are worker-supplied hostnames, so the table must be bounded: a
#: worker rotating its advertised address must not grow hub memory forever.
_RESOLVE_CACHE_MAX = 256


async def _resolve_advertised_host(host: str) -> set[str] | None:
    """Resolve a hostname the way the dialling network would.

    Three-valued by contract: a non-empty set of addresses, an EMPTY set only
    for a definitive does-not-exist answer (EAI_NONAME/EAI_NODATA), and None
    for everything transient — timeout, EAI_AGAIN, resolver trouble — because
    OUR resolver having a bad moment says nothing about the record.

    UnicodeError is in the transient clause because getaddrinfo raises it (a
    ValueError, NOT an OSError) for an IDNA-invalid label — uncaught, it
    escaped to the route's umbrella handler and silently disabled the log-
    signal scan that shares the same try block.
    """
    cached = _RESOLVE_CACHE.get(host)
    if cached and time.monotonic() - cached[0] < _RESOLVE_CACHE_TTL:
        return cached[1]
    try:
        infos = await asyncio.wait_for(asyncio.get_running_loop().getaddrinfo(host, None, type=socket.SOCK_STREAM), 3.0)
        result: set[str] | None = {addr[4][0] for addr in infos}
    except socket.gaierror as exc:
        definitive = {socket.EAI_NONAME, getattr(socket, "EAI_NODATA", socket.EAI_NONAME)}
        result = set() if exc.errno in definitive else None
    except (TimeoutError, OSError, UnicodeError):
        result = None
    now = time.monotonic()
    # Purge expired entries on every write, then cap: without eviction each
    # distinct worker-supplied name would be a permanent key.
    for stale in [k for k, (stamp, _) in _RESOLVE_CACHE.items() if now - stamp >= _RESOLVE_CACHE_TTL]:
        del _RESOLVE_CACHE[stale]
    while len(_RESOLVE_CACHE) >= _RESOLVE_CACHE_MAX:
        del _RESOLVE_CACHE[min(_RESOLVE_CACHE, key=lambda k: _RESOLVE_CACHE[k][0])]
    _RESOLVE_CACHE[host] = (now, result)
    return result


@app.get("/api/services/{slug}/producer-state")
async def api_producer_state(request: Request, slug: str, worker_id: int | None = None) -> dict[str, Any]:
    """Is this service actually EARNING, as distinct from merely running?

    Container health is computed from restarts and crashes, so a service that
    has produced nothing for a month still scores full marks. This is a separate
    verdict, and it says "unknown" rather than guessing when the earnings cannot
    be seen at all.
    """
    _require_auth_api(request)
    service = catalog.get_service(slug)
    if not service:
        raise HTTPException(status_code=404, detail=f"Unknown service '{slug}'")

    from app.collectors import COLLECTOR_MAP

    has_collector = slug in COLLECTOR_MAP

    earned_recently: bool | None = None
    if has_collector:
        # A zero here has three possible causes and only one of them is "idle".
        #
        # get_earned_by_platform sums DELTAS, so a service with a single reading
        # contributes nothing — within an hour of a healthy first install this
        # reported "Recorded earnings have not moved recently" about a service
        # that had produced one perfectly good reading and had had no chance to
        # move. And the sum is in USD, so a platform whose readings cannot be
        # priced sums to zero forever: a MystNodes balance climbing 40 -> 55 ->
        # 70 MYST with no rate available was reported idle indefinitely.
        #
        # Both are "we cannot see", not "it is not earning" — and this module's
        # own docstring says reporting the second when the truth is the first is
        # exactly the false confidence it exists to remove.
        history = await database.get_balance_history(slug, days=7)
        if len(history) >= 2:
            priceable = all(
                str(row.get("currency") or "USD").upper() == "USD" or row.get("fx_rate_usd") for row in history
            )
            if priceable:
                earned = await database.get_earned_by_platform(days=7)
                earned_recently = float(earned.get(slug) or 0.0) > 0

    # None, not True. If the container lookup below throws, this value is what
    # survives, and claiming "running" on no evidence is exactly the assumption
    # this whole module exists to refuse: assess() would skip its unknown path
    # and, on a still-cached earnings figure, report the service as PRODUCING
    # at a moment when we cannot tell whether the container is even up.
    running: bool | None = None
    log_hits: list[dict[str, str]] = []
    traffic: str | None = None
    address_mismatch: str | None = None
    signals = producer_state.signals_for(service)
    try:
        containers = await _get_all_worker_containers()
        # Heartbeat entries key the service as "slug"; matching on "service"
        # here found nothing in production while the tests, which hand-fed
        # the wrong shape, passed.
        matches = [c for c in containers if egress.container_slug(c) == slug]
        running = any(str(c.get("status", "")).lower() == "running" for c in matches)
        traffic = _traffic_state(slug, matches)
        if running:
            # Own guard, not just the outer umbrella: an exception here would
            # otherwise abandon the whole try block and silently zero the
            # log-signal scan below — the address check disabling the very
            # detection (#318) that shares this block. No claim, ever, at the
            # cost of the rest of the verdict.
            with contextlib.suppress(Exception):
                address_mismatch = await _advertised_address_mismatch(matches, worker_id)
        if signals and running:
            wid = worker_id if worker_id is not None else (matches[0].get("_worker_id") if matches else None)
            if wid is not None:
                # The sweep's verdicts first, so a failed tail read below cannot drop them.
                up = {
                    int(c.get("instance") or 1)
                    for c in matches
                    if c.get("_worker_id") == wid and str(c.get("status", "")).lower() == "running"
                }
                log_hits = [
                    {"pattern": "", "means": means, "state": producer_state.FAILING}
                    for (s_slug, s_wid, s_instance), hints in _windowed_hints.items()
                    if s_slug == slug and s_wid == wid and s_instance in up
                    for means in hints
                ]
                logs = (await _proxy_worker_logs(wid, slug, lines=200)).get("logs", "")
                log_hits += producer_state.match_log_signals(logs, signals)
    except Exception as exc:
        # Never let a log or worker problem turn "is it earning?" into a 500;
        # the earnings signal alone is still worth reporting.
        logger.warning("Producer-state signals unavailable for %s: %s", slug, exc)

    return producer_state.assess(
        slug=slug,
        has_collector=has_collector,
        earned_recently=earned_recently,
        log_hits=log_hits,
        traffic=traffic,
        container_running=running,
        address_mismatch=address_mismatch,
    )


@app.get("/api/services/{slug}/disclosure")
async def api_service_disclosure(request: Request, slug: str) -> dict[str, Any]:
    """What this service does with your machine, and what nobody has answered yet."""
    _require_auth_api(request)
    service = catalog.get_service(slug)
    if not service:
        raise HTTPException(status_code=404, detail=f"Unknown service '{slug}'")
    return disclosure.for_service(service)


@app.post("/api/services/{slug}/test-credentials")
async def api_test_credentials(request: Request, slug: str) -> dict[str, Any]:
    """Check the saved credentials for one service, right now.

    Without this a user waits up to a full collection interval — an hour — to
    find out a pasted token had a typo, and learns it from a notification bell.

    The response deliberately has no field that could carry a secret: outcomes
    are classified, and neither the credential nor the provider's raw body is
    returned. A failed provider login frequently echoes the submitted payload.
    """
    from app import collectors
    from app.collectors import COLLECTOR_MAP
    from app.collectors.base import classify_exception

    # Owner, not merely authenticated: this fires an authenticated login to a
    # third party using the OWNER's stored credentials and reads the balance
    # back. /api/collectors/meta already requires owner for the same material,
    # and POST /api/collect — which is strictly less privileged — requires
    # writer. A viewer must not be able to reach any of it.
    _require_owner(request)
    service = catalog.get_service(slug)
    if not service:
        raise HTTPException(status_code=404, detail=f"Unknown service '{slug}'")
    name = service.get("name") or slug

    if slug not in COLLECTOR_MAP:
        return credential_test.result(credential_test.UNSUPPORTED, name)

    remaining = credential_test.cooldown_remaining(slug)
    if remaining > 0:
        # Retrying a rejected login in a tight loop is how accounts get flagged,
        # and a button invites exactly that.
        return credential_test.result(credential_test.RATE_LIMITED, name, retry_after=round(remaining))

    config = await database.get_config() or {}
    collector, missing = collectors.build_one(slug, config)
    if collector is None:
        outcome = credential_test.NOT_CONFIGURED if missing else credential_test.UNSUPPORTED
        return credential_test.result(outcome, name)

    # Re-check immediately before claiming the slot. The first check above is
    # separated from this point by `await database.get_config()`, and an await
    # is where the event loop can run the other request: two clicks could both
    # pass the cooldown, both build a collector, and both reach the provider.
    # Hammering a provider with repeated logins is what gets accounts flagged,
    # which is the entire reason the cooldown exists. `build_one` is
    # synchronous, so re-check and claim are adjacent with no await between
    # them and the window is closed without needing a lock.
    remaining = credential_test.cooldown_remaining(slug)
    if remaining > 0:
        return credential_test.result(credential_test.RATE_LIMITED, name, retry_after=round(remaining))
    credential_test.note_attempt(slug)
    try:
        await _attach_node_readings(collector)
        result = await collector.collect()
    except Exception as exc:
        logger.debug("Credential test for %s raised: %s", slug, exc)
        from app.collectors.base import describe_exception

        # str(exc) is "" for a timeout, which classify() reads as success.
        return credential_test.result(credential_test.classify(describe_exception(exc), classify_exception(exc)), name)
    finally:
        with contextlib.suppress(Exception):
            await collector.close()

    # The collector's own mark first: "Session expired" or a 503 carry it
    # where their text matches nothing the classifier knows.
    outcome = credential_test.classify(result.error, result.error_kind)
    if outcome == credential_test.OK:
        # An OK with an error (the login worked, no figure could be read) has
        # no balance to report: its 0.0 is a placeholder, not a reading.
        balance = None if result.error else result.balance
        return credential_test.result(outcome, name, balance=balance, currency=result.currency)
    logger.debug("Credential test for %s failed: %s", slug, result.error)
    return credential_test.result(outcome, name)


@app.get("/api/earnings/payouts")
async def api_payouts(request: Request, platform: str | None = None) -> dict[str, Any]:
    """Payouts, split into confirmed income and drops still awaiting a human."""
    _require_auth_api(request)
    rows = await database.get_payouts(platform=platform)
    return {
        "confirmed": [r for r in rows if r.get("confirmed")],
        "probable": [r for r in rows if not r.get("confirmed")],
    }


async def _payout_platform(payout_id: int) -> str | None:
    """Which platform a payout row belongs to, before it is deleted."""
    try:
        rows = await database.get_payouts()
    except Exception as exc:  # noqa: BLE001 - answering the payout must not 500 on the lookup
        # Silent None here left the bell entry standing forever with no trace
        # of why; the operator "answered" the prompt and it never went away.
        logger.warning("Could not resolve payout %s to a platform; its bell entry stays: %s", payout_id, exc)
        return None
    for row in rows:
        if row.get("id") == payout_id:
            return row.get("platform")
    return None


async def _retire_payout_alert(payout_id: int, platform: str | None = None) -> None:
    """Drop the bell entry for a payout that has now been answered."""
    global _collector_alerts
    if platform is None:
        platform = await _payout_platform(payout_id)
    if not platform:
        return
    try:
        await database.clear_alerts("payout", platform)
    except Exception as exc:  # noqa: BLE001
        # Do NOT prune the in-memory bell when the durable delete failed: the
        # two stores would diverge on a financial record — the operator sees
        # the prompt disappear, believes it answered, and the surviving row
        # resurrects it after the next restart.
        logger.warning("Could not clear the stored payout alert for %s; keeping it in the bell: %s", platform, exc)
        return
    _collector_alerts = [
        a for a in _collector_alerts if not (a.get("kind") == "payout" and a.get("platform") == platform)
    ]


class ManualPayout(BaseModel):
    """A cash-out the user records by hand."""

    platform: str = Field(min_length=1, max_length=64)
    amount: float = Field(gt=0, allow_inf_nan=False)
    #: Defaults to the unit the service's balance is recorded in.
    currency: str | None = Field(default=None, max_length=12)
    method: str = Field(default="", max_length=40)
    #: YYYY-MM-DD; defaults to today.
    date: str | None = None

    @field_validator("date")
    @classmethod
    def _past_iso_date(cls, value: str | None) -> str | None:
        if value in (None, ""):
            return None
        value = value.strip()
        if not _ISO_DATE.match(value):
            raise ValueError("date must be YYYY-MM-DD")
        try:
            day = datetime.strptime(value, "%Y-%m-%d").date()  # noqa: DTZ007 -- a calendar date
        except ValueError as exc:
            raise ValueError("date must be a real calendar date") from exc
        if day > datetime.now(UTC).date():
            raise ValueError("date cannot be in the future")
        return value


@app.post("/api/earnings/payouts")
async def api_add_payout(request: Request, body: ManualPayout) -> dict[str, Any]:
    """Record a cash-out CashPilot did not detect, so lifetime earnings keep it."""
    _require_writer(request)
    if not catalog.get_service(body.platform):
        raise HTTPException(status_code=404, detail=f"Unknown service '{body.platform}'")
    currency = (body.currency or "").strip().upper()
    if not currency:
        history = await database.get_balance_history(body.platform, days=3650)
        currency = (
            (history[-1].get("currency") if history else None)
            or ((catalog.get_service(body.platform) or {}).get("cashout") or {}).get("currency")
            or "USD"
        )
    fx = 1.0 if currency == "USD" else exchange_rates.to_usd(1.0, currency)
    if currency != "USD" and body.date and body.date != datetime.now(UTC).date().isoformat():
        # Only today's quote exists. A past cash-out priced at it would be restated
        # by today's price, so it stays unpriced and is counted as such.
        fx = None
    try:
        payout_id = await database.add_manual_payout(
            platform=body.platform,
            amount=body.amount,
            currency=currency,
            fx_rate_usd=fx,
            method=body.method.strip(),
            paid_on=body.date,
        )
    except database.PendingPayoutError as exc:
        raise HTTPException(
            status_code=409,
            detail="CashPilot detected a cash-out for this service that is waiting in the alerts. "
            "Confirm or reject it first, so the same cash-out is not counted twice.",
        ) from exc
    return {"ok": True, "id": payout_id, "currency": currency, "priced": fx is not None}


@app.delete("/api/earnings/payouts/{payout_id}")
async def api_delete_payout(request: Request, payout_id: int) -> dict[str, Any]:
    """Remove a confirmed payout recorded by mistake."""
    _require_writer(request)
    if not await database.delete_confirmed_payout(payout_id):
        raise HTTPException(status_code=404, detail="No confirmed payout with that id")
    return {"ok": True, "id": payout_id, "removed": True}


@app.post("/api/earnings/payouts/{payout_id}/confirm")
async def api_confirm_payout(request: Request, payout_id: int, method: str = "") -> dict[str, Any]:
    """Confirm a drop really was a payout. Only a human reaches this."""
    # Writer, not viewer: this mutates financial records, and rejection is a
    # hard DELETE. A read-only account must not be able to destroy them.
    _require_writer(request)
    # Forward `method`. The endpoint has always accepted it and the database
    # has always had a column for it, but it was never passed along, so every
    # confirmation recorded an empty method however the caller was paid.
    if not await database.confirm_payout(payout_id, method=method):
        raise HTTPException(status_code=404, detail="No unconfirmed payout with that id")
    # The question has been answered, so retire the prompt. Left behind, the
    # stored alert would be restored on the next restart and ask again about a
    # payout the user already confirmed.
    await _retire_payout_alert(payout_id)
    return {"ok": True, "id": payout_id, "confirmed": True}


@app.post("/api/earnings/payouts/{payout_id}/reject")
async def api_reject_payout(request: Request, payout_id: int) -> dict[str, Any]:
    """This drop was not a payout — forget it entirely."""
    # Writer, not viewer: this mutates financial records, and rejection is a
    # hard DELETE. A read-only account must not be able to destroy them.
    _require_writer(request)
    platform = await _payout_platform(payout_id)
    if not await database.reject_payout(payout_id):
        raise HTTPException(status_code=404, detail="No unconfirmed payout with that id")
    await _retire_payout_alert(payout_id, platform=platform)
    return {"ok": True, "id": payout_id, "removed": True}


@app.get("/api/services/{slug}/payout-progress")
async def api_payout_progress(request: Request, slug: str) -> dict[str, Any]:
    """Current balance, lifetime earned, and how far off the payout is.

    These are three different questions and were previously one number that went
    DOWN when the user got paid. The projection is the answer to the most
    demotivating unknown in this category — a 20 USD minimum can be months on
    one device, and not knowing that is what makes people give up.
    """
    _require_auth_api(request)
    service = catalog.get_service(slug)
    if not service:
        raise HTTPException(status_code=404, detail=f"Unknown service '{slug}'")

    balance = await database.get_latest_balance(slug)
    history = await database.get_balance_history(slug, days=30)
    # The unit the balance is recorded in, taken from the newest reading we
    # already fetched rather than a second query. None when nothing has ever
    # been collected, which min_payout_in reads as "cannot reconcile" and so
    # leaves the declared minimum alone.
    balance_currency = (history[-1].get("currency") if history else None) or None
    confirmed = await database.get_payouts(platform=slug, confirmed_only=True)
    # A provider read node by node reports its own lifetime, settlements
    # included (issue #488); balance plus confirmed payouts would miss every
    # settlement, since none is ever asked about.
    node_total = (await _node_totals()).get(slug)

    known = balance is not None
    current = float(balance or 0.0)
    lifetime = payouts.lifetime_earned(current, confirmed) if (known or confirmed) else None
    # The card renders every figure here in ONE unit and asks the user to compare
    # them against the provider's minimum. The balance is in whatever the
    # collector reports and the minimum in whatever the provider cashes out in;
    # for Storj and anyone-protocol those differ, so "3.50 STORJ" was really
    # $3.50 and "0.50 to go" counted down to a threshold in another unit.
    # Expressing the minimum in the BALANCE's unit keeps the comparison honest
    # and leaves the balance itself untouched.
    minimum = payouts.min_payout_in(service, balance_currency)
    declared_currency = payouts.min_payout_currency(service)
    comparable = minimum is not None or payouts.min_payout(service) is None
    return {
        "slug": slug,
        "current_balance": round(current, 6) if known else None,
        # What the numbers above are actually in, so nothing has to assume.
        "balance_currency": balance_currency,
        "min_amount": minimum,
        # Falls back to the DECLARED unit when nothing has been collected: with
        # no balance currency, min_payout_in returns the catalog figure at face
        # value, and labelling that with None leaves the consumer holding a
        # threshold it cannot attribute to any unit.
        "min_amount_currency": (balance_currency if (minimum is not None and balance_currency) else declared_currency),
        # False only when a real minimum exists and could not be brought into
        # the balance's unit. The card hides the comparison rather than drawing
        # a bar out of two different currencies.
        "comparable": comparable,
        # Lifetime counts CONFIRMED payouts only. A probable one folded in here
        # would let a single misread drop inflate earnings forever, invisibly.
        #
        # None when the balance is unknown AND nothing has been paid out:
        # `float(balance or 0.0)` folded "never read" into a real zero, so the
        # card stated a definite 0.00 lifetime for a service nothing had ever
        # looked at. Confirmed payouts are still a real lower bound, so they
        # keep a figure.
        "lifetime_earned": node_total["lifetime"] if node_total else lifetime,
        "confirmed_payout_count": len(confirmed),
        "balance_known": known,
        # Projection stays. payouts.project already returns an explicit
        # NOT_ENOUGH_DATA state, which is a more useful answer than null — it
        # says WHY there is no estimate. Nulling it discarded that and pushed a
        # null-check onto every caller. The misleading part was never this
        # field; it was the card rendering a 0%-of-minimum bar from it, which
        # is fixed where the card is drawn.
        "projection": payouts.project(current, service, history, balance_currency),
    }


@app.get("/api/fleet/economics")
async def api_fleet_economics(request: Request) -> dict[str, Any]:
    """Is each machine worth keeping powered on for what it earns?

    Deliberately per-MACHINE. Adding a bandwidth container to a box that is
    already on costs 1-3 W — below what a consumer smart plug can measure — so a
    per-service net figure would be fabricated precision. The machine-level
    question is the one with an answer: a dedicated 65 W node at 0.20/kWh costs
    about 9.50 a month against a typical 3-6 gross.

    Nothing is ever stopped or throttled on the strength of this. The
    electricity is the operator's and so is the decision.
    """
    _require_auth_api(request)
    config = await database.get_config() or {}
    try:
        # Two features shipped in one release reading DIFFERENT keys for the
        # same tariff, so setting one left the other reporting "cost unknown"
        # forever — and both unknown-paths are deliberately quiet, which is
        # exactly what hid it. power_price_per_kwh is canonical (it shipped
        # first); the newer name is honoured so nobody's existing config breaks.
        price = _tariff_price(config)
    except (TypeError, ValueError):
        price = 0.0

    # The tariff is in the user's own currency; every gross here comes from
    # get_earned_by_platform, which is USD by contract. Subtracting one from the
    # other produced a "losing money — turning it off would save that" verdict
    # off by the whole FX spread, on a payload that carried no currency label at
    # all. Convert the TARIFF, so the endpoint stays canonical USD like the rest
    # of the API and the frontend's display-currency layer can render it.
    tariff_currency = str(config.get("power_currency") or "EUR")
    fx_ok = True
    if price and tariff_currency != "USD":
        converted = exchange_rates.to_usd(price, tariff_currency)
        if converted is None:
            fx_ok = False
        else:
            price = converted

    # Fetched once and reused: this used to query the worker table here and
    # then again inside _get_all_worker_containers, decoding every row twice
    # to build the same list in a single request.
    raw_workers = await database.list_workers()
    workers = [_decoded_worker(w) for w in raw_workers]
    earned = await database.get_earned_by_platform(days=30)
    containers = await _get_all_worker_containers(raw_workers)

    # Attribute each service's gross to the worker running it. A service on two
    # machines splits evenly: without per-node earnings there is no better
    # answer, and pretending otherwise would be invented precision again.
    per_worker_gross: dict[Any, float] = {}
    hosts: dict[str, list[Any]] = {}
    for container in containers:
        slug = egress.container_slug(container)
        if slug:
            hosts.setdefault(slug, []).append(container.get("_worker_id"))
    for slug, worker_ids in hosts.items():
        share = float(earned.get(slug) or 0.0) / max(1, len(worker_ids))
        for worker_id in worker_ids:
            per_worker_gross[worker_id] = per_worker_gross.get(worker_id, 0.0) + share

    assessed = []
    for worker in workers:
        info = worker.get("system_info") or {}
        # Keyed on client_id, not the autoincrement row id.
        #
        # delete_worker removes the row and nothing else, so a host that is
        # removed and re-enrols gets a fresh id — orphaning its watts and
        # dedicated settings silently, while the old keys linger forever. The
        # id is also read from the row-id fallback for volumes written before
        # this change.
        raw_watts = config.get(f"worker_{_worker_config_key(worker)}_watts") or config.get(
            f"worker_{worker.get('id')}_watts"
        )
        try:
            watts = float(raw_watts) if raw_watts else None
        except (TypeError, ValueError):
            watts = None
        assessed.append(
            machine_economics.assess_machine(
                name=worker.get("name") or f"worker {worker.get('id')}",
                # None, not 0.0, when this worker's containers were never
                # counted. per_worker_gross is built only from ONLINE workers
                # (_get_all_worker_containers skips the rest), so defaulting an
                # offline machine to zero earnings is what produced the
                # "turn it off" advice about a host we cannot see.
                monthly_gross=(
                    per_worker_gross.get(worker.get("id"), 0.0) if str(worker.get("status") or "") == "online" else None
                ),
                watts=watts,
                # No rate means the tariff cannot be expressed in the same unit
                # as the earnings, so there is no honest cost. None is already
                # this module's "unknown", and it reports that rather than
                # inventing a net.
                price_per_kwh=(price or None) if fx_ok else None,
                metered=power.is_metered(info),
                dedicated=_worker_flag(config, worker, "dedicated"),
            )
        )

    summary = machine_economics.fleet_summary(assessed)
    # Every figure in this payload is USD. It previously carried no currency at
    # all, so fleet.html printed bare numbers the user could not attribute to a
    # unit — and the numbers themselves mixed two.
    summary["currency"] = "USD"
    if not fx_ok:
        summary["fx_unavailable"] = True
        summary["tariff_currency"] = tariff_currency
        summary["cost_unavailable_reason"] = (
            f"Your electricity tariff is in {tariff_currency} and earnings are recorded in "
            f"USD. No {tariff_currency}-to-USD rate is available right now, so a running "
            "cost would be two different currencies subtracted from each other."
        )
    return summary


def _is_phone(system_info: dict[str, Any] | None) -> bool:
    """An Android worker, by either field a client sets.

    The current Android client reports ``os: "Android"`` and no device_type, so
    checking device_type alone let phones into the usage table as rows of
    nothing but dashes: a phone has no Docker stats to give.
    """
    return placement.is_android(system_info or {})


def _usage_point(rows: list[dict[str, Any]]) -> dict[str, Any]:
    """One point (an hour, or 6 on 30 days) of one server or of one container.

    CPU and memory are each container's average while it was read, weighted by
    the share of the point it ran (its heartbeats over the most any container
    had), then added up: a container that ran one hour of six adds a sixth of
    its load, not all of it. A running container whose stats read failed leaves
    the sum a floor (``cpu_partial`` / ``memory_partial``), never a silent drop.
    Traffic is each container's bytes over the seconds its deltas cover, added
    up as a rate. Unknown stays None: an hour with no rows is a gap, and a
    container with no counters (host network) leaves the traffic a floor
    (``traffic_partial``), never a total that silently misses it.
    """
    point: dict[str, Any] = {
        "cpu_percent": None,
        "memory_mb": None,
        "rx_rate": None,
        "tx_rate": None,
        "rx_bytes": None,
        "tx_bytes": None,
        "traffic_partial": False,
        "cpu_partial": False,
        "memory_partial": False,
    }

    def add(key: str, value: Any) -> None:
        if value is not None:
            point[key] = value if point[key] is None else point[key] + value

    full = max((r["beats"] for r in rows), default=0)
    for r in rows:
        share = r["beats"] / full if full else 1.0
        if r["cpu_n"]:
            add("cpu_percent", r["cpu_sum"] / r["cpu_n"] * share)
        if r["mem_n"]:
            add("memory_mb", r["mem_sum"] / r["mem_n"] * share)
        if r["cpu_n"] < r["beats"]:
            point["cpu_partial"] = True
        if r["mem_n"] < r["beats"]:
            point["memory_partial"] = True
        rx_rate = usage.rate(r["rx_bytes"], r["net_seconds"])
        tx_rate = usage.rate(r["tx_bytes"], r["net_seconds"])
        if rx_rate is None or tx_rate is None or r["no_counters"] or r["traffic_lost"]:
            point["traffic_partial"] = True
        add("rx_rate", rx_rate)
        add("tx_rate", tx_rate)
        if rx_rate is not None and tx_rate is not None:
            add("rx_bytes", r["rx_bytes"])
            add("tx_bytes", r["tx_bytes"])
    for key in ("cpu_percent", "rx_rate", "tx_rate"):
        if point[key] is not None:
            point[key] = round(point[key], 2)
    if point["memory_mb"] is not None:
        point["memory_mb"] = round(point["memory_mb"], 1)
    return point


@app.get("/api/usage")
async def api_usage(
    request: Request, days: int = 7, worker_id: int | None = None, container: str | None = None
) -> dict[str, Any]:
    """Hourly CPU, memory and traffic of one server, or of one of its containers.

    ``servers`` feeds the two pickers: every server (phones left out, they have
    no Docker stats) and the containers it reported in the window. ``hours``
    is the start of every point in the window, oldest first: one per hour on
    7 days, one per ``step`` (6 hours) on 30, where 720 hourly points would be
    noise on a small chart. Each series has one value per point; None is "no
    reading", drawn as a gap, never as 0.
    """
    _require_auth_api(request)
    days = min(max(int(days), 1), 30)
    step = 3600 if days <= 7 else 6 * 3600
    now_point = int(time.time() // step) * step
    hours = [now_point - i * step for i in range(days * 86400 // step - 1, -1, -1)]
    workers = {
        w["id"]: w for w in await database.list_workers() if not _is_phone(_safe_json(w.get("system_info", "{}"), {}))
    }
    rows = [r for r in await database.get_usage_hourly(hours[0]) if r["worker_id"] in workers]

    by_worker: dict[int, dict[str, dict[str, Any]]] = {}
    for r in rows:
        seen = by_worker.setdefault(r["worker_id"], {}).setdefault(
            r["container"], {"name": r["container"], "slug": r["slug"], "no_counters": False}
        )
        seen["slug"] = r["slug"]
        seen["no_counters"] = seen["no_counters"] or bool(r["no_counters"])
    servers = []
    for wid, w in sorted(workers.items(), key=lambda kv: (kv[1].get("name") or "").lower()):
        containers = sorted(by_worker.get(wid, {}).values(), key=lambda c: c["name"])
        for c in containers:
            c["service"] = (catalog.get_service(c["slug"]) or {}).get("name") or c["slug"]
        servers.append(
            {
                "worker_id": wid,
                "name": w.get("name") or f"worker {wid}",
                "status": w.get("status"),
                "containers": containers,
            }
        )
    if worker_id not in workers:
        with_data = [s for s in servers if s["containers"]]
        worker_id = (with_data or servers or [{"worker_id": None}])[0]["worker_id"]
    if container is not None and container not in by_worker.get(worker_id, {}):
        container = None

    # One merged row per container and point: sums and counts add, so a 6-hour
    # point is the average over the hours that had readings.
    merged: dict[tuple[int, str], dict[str, Any]] = {}
    for r in rows:
        if r["worker_id"] != worker_id or (container is not None and r["container"] != container):
            continue
        point = r["hour"] // step * step
        m = merged.get((point, r["container"]))
        if m is None:
            merged[(point, r["container"])] = dict(r)
            continue
        for key in ("beats", "cpu_sum", "cpu_n", "mem_sum", "mem_n", "net_seconds", "no_counters", "traffic_lost"):
            m[key] += r[key]
        for key in ("rx_bytes", "tx_bytes"):
            if r[key] is not None:
                m[key] = r[key] if m[key] is None else m[key] + r[key]
    per_hour: dict[int, list[dict[str, Any]]] = {}
    for (point, _), m in merged.items():
        per_hour.setdefault(point, []).append(m)
    points = [_usage_point(per_hour[h]) if h in per_hour else None for h in hours]
    keys = (
        "cpu_percent",
        "memory_mb",
        "rx_rate",
        "tx_rate",
        "rx_bytes",
        "tx_bytes",
        "traffic_partial",
        "cpu_partial",
        "memory_partial",
    )
    series = {k: [None if p is None else p[k] for p in points] for k in keys}
    return {
        "days": days,
        "step": step,
        "hours": hours,
        "servers": servers,
        "worker_id": worker_id,
        "container": container,
        "series": series,
    }


@app.get("/api/services/{slug}/deploy-risk")
async def api_deploy_risk(request: Request, slug: str) -> dict[str, Any]:
    """What the user should know BEFORE deploying this, in plain words.

    The documented risk of proxyware is not container escape — it is that
    someone else's traffic exits the user's address and is attributed to them.
    That belongs at the deploy step, not buried in an FAQ: a self-hosting
    audience is persuaded by a project that owns the downside.

    The notice is driven by the service's own disclosure entry, so it fires
    where strangers really do route through the user's IP and stays silent for a
    storage node, where it would be false and would train people to click past
    warnings that matter.
    """
    _require_auth_api(request)
    service = catalog.get_service(slug)
    if not service:
        raise HTTPException(status_code=404, detail=f"Unknown service '{slug}'")

    return {
        "slug": slug,
        "attribution": lan_isolation.attribution_notice(service),
        "isolation": lan_isolation.assess(service),
    }


@app.get("/api/fleet/isolation-guide")
async def api_isolation_guide(request: Request) -> dict[str, Any]:
    """Which services can be LAN-isolated, and what each one needs allowed.

    Returns the recipe rather than applying it. Creating bridges and firewall
    rules on someone's host is a change with real blast radius, and several
    services break in ways that cost money when the exceptions are missed.
    """
    _require_auth_api(request)
    assessed = [lan_isolation.assess(s) for s in catalog.get_services()]
    return {
        "network_name": lan_isolation.DEFAULT_NETWORK_NAME,
        "blocked_destinations": list(lan_isolation.RFC1918 + lan_isolation.LINK_LOCAL),
        "compose_snippet": lan_isolation.compose_snippet(),
        "isolatable": [a["slug"] for a in assessed if a["verdict"] == lan_isolation.ISOLATABLE],
        "needs_exceptions": [a for a in assessed if a["verdict"] == lan_isolation.NEEDS_EXCEPTIONS],
        "not_isolatable": [a for a in assessed if a["verdict"] == lan_isolation.NOT_ISOLATABLE],
    }


@app.get("/api/disclosure/coverage")
async def api_disclosure_coverage(request: Request) -> dict[str, Any]:
    """How much of the catalog is documented, and which services are not.

    Deliberately reports the gap: presenting the documented subset without
    saying what is missing would imply a completeness the catalog does not have.
    """
    _require_auth_api(request)
    return disclosure.coverage(catalog.get_services())


@app.get("/api/payout-registry")
async def api_payout_registry(request: Request) -> dict[str, Any]:
    """Where every service pays out (CashPilot-luj, tier 1).

    OWNER-ONLY, deliberately. The addresses themselves are public data, but this
    response maps one person to all of their wallets at once -- exactly the
    correlation an ordinary viewer has no business being handed.
    """
    _require_owner(request)
    return await payout_registry.registry(read_addresses=_read_payout_addresses)


async def _backup_box(slug: str, worker_id: Any) -> dict[str, Any]:
    """The box running ``slug`` that a backup confirmation or export is about.

    Refuses a service whose catalog declares nothing irreplaceable, and a box
    that does not report the service: a confirmation for either would be a
    confirmation of nothing.
    """
    svc = catalog.get_service(slug)
    if not svc or not payout_registry.critical_state(svc):
        raise HTTPException(status_code=404, detail="This service keeps no irreplaceable state to back up.")
    if isinstance(worker_id, bool) or not isinstance(worker_id, int):
        raise HTTPException(status_code=400, detail="worker_id must be a number.")
    workers = await database.list_workers()
    for row in placement.runs_on(workers).get(slug, []):
        if row["worker_id"] == worker_id and not row["android"]:
            return {"worker_id": worker_id, "name": row["name"], "workers": workers}
    raise HTTPException(status_code=404, detail="That box does not run this service.")


async def _json_object(request: Request) -> dict[str, Any]:
    """The request body as a JSON object.

    Read by hand rather than through a pydantic model: a 422 from a model echoes
    the offending input, and the input here can be a passphrase.
    """
    try:
        body = await request.json()
    except ValueError:
        body = None
    if not isinstance(body, dict):
        raise HTTPException(status_code=400, detail="Expected a JSON object.")
    return body


@app.post("/api/payout-registry/{slug}/backup-confirm")
async def api_backup_confirm(slug: str, request: Request) -> dict[str, Any]:
    """The owner says this box's copy of ``slug`` is backed up.

    Recorded with the state fingerprint the box reports now, so a new volume
    later voids it. OWNER-ONLY, like the page.
    """
    _require_owner(request)
    body = await _json_object(request)
    box = await _backup_box(slug, body.get("worker_id"))
    current = payout_registry.state_ids(box["workers"]).get((box["worker_id"], slug))
    await database.confirm_state_backup(slug, box["worker_id"], current, "manual")
    return {"ok": True}


@app.delete("/api/payout-registry/{slug}/backup-confirm")
async def api_backup_unconfirm(slug: str, request: Request) -> dict[str, Any]:
    """Take a confirmation back: the box shows as not backed up again.

    ``worker_id`` is read after the owner check, so an anonymous caller gets
    401 rather than a validation error.
    """
    _require_owner(request)
    try:
        worker_id = int(request.query_params.get("worker_id", ""))
    except ValueError:
        raise HTTPException(status_code=400, detail="worker_id must be a number.") from None
    await database.clear_state_backup(slug, worker_id)
    return {"ok": True}


@app.post("/api/payout-registry/{slug}/backup-export")
async def api_backup_export(slug: str, request: Request) -> dict[str, Any]:
    """Download an encrypted backup of ``slug``'s irreplaceable state on one box.

    The worker reads only the catalog's ``critical_volumes`` and encrypts there,
    to the recipient public key or passphrase the owner gives here; this
    installation keeps neither, so it cannot open what it exports. The bundle
    comes back in this response and goes nowhere else. A successful export
    counts as a confirmed backup of that box.

    Never logs or echoes the passphrase: the body is read by hand, and the only
    errors returned are the worker's own, which never contain it.
    """
    _require_owner(request)
    body = await _json_object(request)
    passphrase = body.get("passphrase")
    recipient = body.get("recipient_public_key")
    if not isinstance(passphrase, str | None) or not isinstance(recipient, str | None):
        raise HTTPException(status_code=400, detail="The passphrase and the public key must be text.")
    if bool(passphrase) == bool(recipient):
        raise HTTPException(status_code=400, detail="Give either a passphrase or a recipient public key, not both.")
    if passphrase:
        # A typo would give a file nobody can open, and the export below marks
        # the box as backed up. Compared, never echoed.
        repeat = body.get("passphrase_repeat")
        if not isinstance(repeat, str) or not secrets.compare_digest(repeat.encode(), passphrase.encode()):
            raise HTTPException(status_code=400, detail="The two passphrases differ. Type it again.")
    box = await _backup_box(slug, body.get("worker_id"))
    worker_id = box["worker_id"]

    update = "This worker is too old to export backups. Update the worker first."
    if "state_backup" not in await _worker_features(worker_id):
        raise HTTPException(status_code=409, detail=update)
    try:
        result = await _proxy_to_worker(
            worker_id,
            "POST",
            f"/api/containers/{slug}/backup",
            json={"passphrase": passphrase} if passphrase else {"recipient_public_key": recipient},
            timeout=180,
        )
    except HTTPException as exc:
        if exc.status_code == 404:
            raise HTTPException(status_code=409, detail=update) from exc
        raise
    bundle = result.get("bundle_b64") if isinstance(result, dict) else None
    if not isinstance(bundle, str) or not bundle:
        raise HTTPException(status_code=502, detail="The worker answered without a backup.")

    fingerprint = result.get("state_id")
    if not isinstance(fingerprint, str) or not fingerprint:
        fingerprint = payout_registry.state_ids(box["workers"]).get((worker_id, slug))
    await database.confirm_state_backup(slug, worker_id, fingerprint, "export")
    box_name = re.sub(r"[^A-Za-z0-9._-]+", "-", str(box["name"])).strip("-") or str(worker_id)
    return {
        "filename": f"cashpilot-{slug}-{box_name}-{datetime.now(UTC):%Y%m%d}.cpbackup",
        "bytes": result.get("bytes"),
        "targets": result.get("targets") or [],
        "bundle_b64": bundle,
    }


async def _read_payout_addresses(worker_id: int) -> list[dict[str, Any]]:
    """Addresses set inside one worker's containers (its /api/payout-addresses).

    A worker from before this endpoint answers 404, which the registry shows as
    "not checked", never as "not set".
    """
    result = await _proxy_to_worker(worker_id, "GET", "/api/payout-addresses", timeout=15)
    rows = result.get("addresses") if isinstance(result, dict) else None
    return rows if isinstance(rows, list) else []


async def _read_node_figures(slug: str) -> list[dict[str, Any]]:
    """What each node of ``slug`` says about its own earnings (issue #488).

    Asks every online worker that runs the service (its /api/node-earnings).
    A worker that does not answer, or one from before the endpoint (404), adds
    nothing: its node keeps the figures it last reported.
    """
    try:
        where = placement.runs_on(await database.list_workers())
    except Exception as exc:  # noqa: BLE001 - no fleet means no readings, not a crash
        logger.warning("Could not read the workers for the %s node read: %s", slug, exc)
        return []
    targets = sorted(
        {r["worker_id"] for r in where.get(slug, []) if r["online"] and not r["android"] and r["status"] == "running"},
        key=str,
    )

    async def _one(worker_id: Any) -> list[dict[str, Any]]:
        try:
            result = await _proxy_to_worker(worker_id, "GET", "/api/node-earnings", timeout=20)
        except Exception as exc:  # noqa: BLE001 - one worker never fails the read
            logger.info("Could not read %s node figures from worker %s: %s", slug, worker_id, exc)
            return []
        rows = result.get("nodes") if isinstance(result, dict) else None
        return [r for r in rows or [] if isinstance(r, dict) and r.get("slug") == slug and not r.get("error")]

    readings: list[dict[str, Any]] = []
    for rows in await asyncio.gather(*(_one(wid) for wid in targets)):
        for row in rows:
            try:
                balance, lifetime = float(row["balance"]), float(row["lifetime"])
            except (KeyError, TypeError, ValueError):
                continue
            if not (math.isfinite(balance) and math.isfinite(lifetime) and 0 <= balance <= lifetime):
                continue
            reading: dict[str, Any] = {"node": str(row.get("node") or ""), "balance": balance, "lifetime": lifetime}
            # The settlement sums are optional: a node whose history could not
            # be read still counts, it only adds nothing to the fee share.
            try:
                net, fees = float(row["settled_net"]), float(row["settled_fees"])
            except (KeyError, TypeError, ValueError):
                net = fees = -1.0
            if math.isfinite(net) and math.isfinite(fees) and net >= 0 and fees >= 0:
                reading.update(settled_net=net, settled_fees=fees)
            readings.append(reading)
    return [r for r in readings if r["node"]]


async def _node_totals() -> dict[str, dict[str, Any]]:
    """Per-node totals (issue #488), or none when they cannot be read.

    Only Mysterium has any, so a failed read must not cost every other
    service its figures: without them the totals fall back to balance plus
    confirmed payouts, as before.
    """
    try:
        return await database.get_node_totals()
    except Exception as exc:  # noqa: BLE001 - the cards must still render
        logger.warning("Could not read the per-node totals: %s", exc)
        return {}


async def _attach_node_readings(collector: Any) -> None:
    """Hand a collector its nodes' own figures, when its catalog entry reads them."""
    slug = getattr(collector, "platform", "")
    if not hasattr(collector, "node_readings"):
        return
    if not isinstance(((catalog.get_service(slug) or {}).get("collector") or {}).get("node_read"), dict):
        return
    collector.node_readings = await _read_node_figures(slug)


@app.get("/api/payout-balances")
async def api_payout_balances(request: Request) -> dict[str, Any]:
    """What is actually AT each payout address (CashPilot-dv6, tier 2).

    OWNER-ONLY, same reasoning as the registry it builds on.

    A SEPARATE endpoint from /api/payout-registry on purpose. The registry is a
    local join and answers instantly; these are network reads against public
    RPCs that can take seconds or time out. Folding them together would make the
    whole payout table hostage to the slowest chain, so the page renders the
    table first and fills these in afterwards.

    Only addresses we actually hold are queried -- an `internal`, `minted` or
    `unknown` service has no address, and asking a public RPC about nothing is
    both pointless and rude.
    """
    _require_owner(request)
    registry = await payout_registry.registry(read_addresses=_read_payout_addresses)

    wanted = [
        row
        for row in registry["entries"]
        if row.get("model") == "external" and row.get("address") and row.get("chain") in onchain.CHAINS
    ]
    results = await onchain.balances([(row["chain"], row["address"]) for row in wanted])

    balances: dict[str, Any] = {}
    for row, result in zip(wanted, results, strict=True):
        # Decimal does not survive JSON. str() keeps every digit, which is the
        # whole reason the reader uses Decimal in the first place -- a float here
        # would undo it at the last step.
        amount = result.get("amount")
        balances[row["slug"]] = {**result, "amount": None if amount is None else str(amount)}

    return {
        "balances": balances,
        # Named so the UI can say "3 of 5 addresses could not be checked"
        # instead of quietly showing fewer rows than it did a moment ago.
        "checked": len(wanted),
        "unreadable": sum(1 for r in results if r.get("state") != onchain.KNOWN),
    }


@app.get("/api/update-status")
async def api_update_status(request: Request) -> dict[str, Any]:
    """Whether a newer CashPilot has been released (CashPilot-w0ss).

    Reads the cached result only -- it never fetches on the request path, so a
    slow or unreachable GitHub cannot make a page load slowly. The scheduler
    refreshes it once a day.

    ``known`` is the honest field. False means we do not know, which is NOT the
    same as up to date, and the UI must not render it as reassurance.
    """
    _require_auth_api(request)
    return update_check.state()


# How stale the collection stamp may grow before it is a problem: 2.5x the
# interval tolerates one slow run and one missed slot before declaring the
# machinery dead — the same flap-resistance reasoning as the worker's window.
_COLLECTION_STALE_FACTOR = 2.5


@app.get("/api/health")
async def api_health(response: Response) -> dict[str, Any]:
    """Health of the UI process itself (no auth — the Docker healthcheck asks).

    Reports whether this process can still DO ITS JOB, not merely whether the
    port answers. The container healthcheck used to be a TCP connect, which the
    kernel's listen backlog completes for a wedged uvicorn, a dead scheduler or
    an unwritable database — every internal failure rendered as "healthy". A
    check that cannot fail for the failure you actually have is not a check.

    Three concrete questions, each a distinct operational failure an operator
    can act on: is the scheduler running, has the collection machinery
    completed a run recently (a wedged collection lock freezes this stamp —
    the skip line used to disguise that as routine), and can the database be
    read. Values are deliberately generic state names, no secrets.
    """
    problems: list[str] = []
    if not scheduler.running:
        problems.append("scheduler stopped")
    stale_after = _COLLECTION_STALE_FACTOR * COLLECT_INTERVAL_MIN * 60
    age = time.monotonic() - _last_collection_finished
    if age > stale_after:
        problems.append(f"no collection completed in {int(age // 60)} minutes")
    try:
        await database.get_config()
    except Exception:  # noqa: BLE001 - the reason goes to the body, not a 500
        problems.append("database unreadable")
    body: dict[str, Any] = {"status": "ok"}
    if problems:
        response.status_code = 503
        body["status"] = "degraded"
        body["problems"] = problems
    return body


@app.get("/api/collector-alerts")
async def api_collector_alerts(request: Request) -> dict[str, Any]:
    """Collector errors from the last run, and whether a run has happened.

    The bell rendered an empty list as "All collectors healthy". On a fresh
    install — or after a restart, before the first hourly collection — nothing
    has been checked, so that affirmative is unearned: it is the same
    absent-equals-true shape this codebase rejects everywhere else. The bell's
    own FAILURE path is written correctly ("Alerts unavailable" rather than
    healthy), which makes the never-ran case the outlier (CashPilot-tb5).

    Returning an object rather than a bare list so "no alerts" and "nothing has
    run" can be told apart at all. The UI is the only consumer and ships with
    this.
    """
    user = _require_auth_api(request)
    # Activity alerts name a box's LAN probing; the Activity page is owner
    # only, so its alerts are too.
    owner = auth.require_role(user, "owner")
    sanitized: list[dict[str, Any]] = []
    for alert in _collector_alerts:
        if not owner and str(alert.get("kind", "")).startswith("activity_"):
            continue
        error_msg = alert.get("error", "")
        clean = error_msg[:_MAX_ALERT_ERROR_LEN]
        if len(error_msg) > _MAX_ALERT_ERROR_LEN:
            clean += "..."
        # `kind` is additive and defaults to "collector", so a frontend that
        # does not read it behaves exactly as before. `category` is the failure
        # TAXONOMY (auth/transient/shape) and is only present when a collector
        # actually classified it -- absent is unknown, never transient.
        entry = {"kind": alert.get("kind", "collector"), "platform": alert["platform"], "error": clean}
        if alert.get("category"):
            entry["category"] = alert["category"]
        if alert.get("kind") == "missing" and "everywhere" in alert:
            # What the dashboard's Forget buttons act on. Absent on an entry
            # restored at startup until the next health check rebuilds it.
            entry["everywhere"] = bool(alert["everywhere"])
            entry["name"] = alert.get("name") or alert["platform"]
            entry["boxes"] = [{"worker_id": b["worker_id"], "name": b["name"]} for b in alert.get("boxes") or []]
        sanitized.append(entry)
    # The staleness flag is what keeps "All collectors healthy" honest: the
    # collected latch below is permanent once set, so after a scheduler death
    # the bell would affirm health forever on the strength of a collection
    # that stopped happening. Decided server-side so the frontend never has
    # to know the interval.
    age = int(time.monotonic() - _last_collection_finished)
    return {
        "alerts": sanitized,
        "collected": _collection_has_run,
        "last_run_age_seconds": age,
        "collection_stale": age > _COLLECTION_STALE_FACTOR * COLLECT_INTERVAL_MIN * 60,
    }


@app.get("/api/exchange-rates")
async def api_exchange_rates(request: Request) -> dict[str, Any]:
    """Return current exchange rates (fiat + crypto) for frontend conversion."""
    _require_auth_api(request)
    return exchange_rates.get_all()


@app.get("/api/services/{slug}/per-node-earnings")
async def api_per_node_earnings(request: Request, slug: str) -> list[dict[str, Any]]:
    """Return per-node earnings for services that support it (e.g. MystNodes)."""
    _require_auth_api(request)
    config = await database.get_config() or {}
    if not isinstance(config, dict):
        config = {}

    # Driven by the catalog, not by a slug comparison. This used to read
    # `if slug == "mysterium"` and import that collector class by name — two
    # pieces of service-specific knowledge in app/, where the repo's own rule
    # says neither belongs. Adding a second service that reports per-node
    # figures meant editing this function; now it means one line of YAML.
    service = catalog.get_service(slug)
    declares_per_node = bool((service.get("collector") or {}).get("per_node_earnings")) if service else False
    if not declares_per_node:
        return []

    # Imported here, matching api_test_credentials: app.collectors pulls in every
    # collector module, and importing it at module scope would drag that whole
    # tree into any process that imports app.main.
    from app import collectors

    collector, missing = collectors.build_one(slug, config)
    if collector is None or missing:
        return []
    try:
        # The capability is declared in YAML, so a service can claim it without
        # its collector implementing it yet. Refuse rather than 500.
        getter = getattr(collector, "get_per_node_earnings", None)
        if getter is None:
            logger.warning("%s declares per_node_earnings but its collector does not implement it", slug)
            return []
        try:
            return await getter()
        except Exception as exc:
            # This call reaches a THIRD-PARTY API: it times out, rate-limits,
            # and returns HTML instead of JSON on a bad day. None of that is a
            # fault in CashPilot, and none of it should reach the user as a raw
            # 500 — every other collector call in this file degrades instead.
            # An empty per-node breakdown alongside the account total is a
            # smaller loss than a broken page.
            logger.warning("Per-node earnings unavailable for %s: %s", slug, exc)
            return []
    finally:
        with contextlib.suppress(Exception):
            await collector.close()


def _config_flag(config: dict[str, Any], key: str) -> bool:
    """A stored config value read as a boolean.

    Config values are TEXT. `bool("false")` is True, so a user who set a flag
    to "false", "0" or "no" got the opposite of what they asked for. Uses the
    same truthy set as database.set_config so a value written by one half of
    the app means the same thing to the other.
    """
    from app.database import _TRUTHY

    return str(config.get(key, "") or "").strip().lower() in _TRUTHY


def _tariff_price(config: dict[str, Any]) -> float:
    """The electricity price per kWh, honouring the legacy key name.

    `electricity_price_per_kwh` was renamed to `power_price_per_kwh`. One
    endpoint accepted both and the other accepted only the new name, so an
    upgrading user who had set the old key saw running costs on the fleet page
    and "cost unknown" on the dashboard, from the same stored value. Both now
    resolve through here.
    """
    return float(config.get("power_price_per_kwh") or config.get("electricity_price_per_kwh") or 0.0)


def _to_usd_with_stored(amount: float, currency: str, stored_rate: float | None) -> float | None:
    """Convert to USD, falling back to the rate the reading was RECORDED at.

    ``exchange_rates.to_usd`` consults only the live caches, so a crypto whose
    rate lookup is merely stale was dropped from the dashboard total entirely —
    silently, with nothing on screen saying the headline figure was incomplete.

    This is not a new rule. ``get_earnings_dashboard_summary`` and
    ``get_earned_by_platform`` already price rows this way, and
    ``upsert_earnings`` stores ``fx_rate_usd`` on every row specifically so
    "the USD value of a historical non-USD reading cannot be reconstructed
    later" without it. The Total was the one figure applying a stricter rule
    than the cards beside it, which is why they disagreed.

    A stored rate is the rate at collection time rather than now — imperfect,
    and much better than omitting the holding. Callers count what still cannot
    be priced so the response can say so.

    Module-level rather than a closure: defined inside the loop it captured
    ``currency`` late, so every conversion would have used the LAST currency
    seen (ruff B023).
    """
    live = exchange_rates.to_usd(amount, currency)
    if live is not None:
        return live
    return amount * stored_rate if stored_rate is not None else None


def _worker_flag(config: dict[str, Any], worker: dict[str, Any], suffix: str) -> bool:
    """A per-worker boolean setting, read by client_id with a row-id fallback.

    Two problems in one place. Config values are TEXT, so ``bool("false")`` is
    True and a user who set a flag to "false" got the opposite of what they
    asked for; ``database._TRUTHY`` is the parser the rest of the app already
    uses. And the key was built from the AUTOINCREMENT row id, which changes
    when a host is removed and re-enrolled — silently orphaning the setting —
    so ``client_id`` is preferred, with the old id-based key still honoured for
    volumes written before this change.
    """
    for key in (f"worker_{_worker_config_key(worker)}_{suffix}", f"worker_{worker.get('id')}_{suffix}"):
        if str(config.get(key, "") or "").strip():
            return _config_flag(config, key)
    return False


def _worker_config_key(worker: dict[str, Any]) -> str:
    """The stable identifier for per-worker config keys.

    The row id is an AUTOINCREMENT primary key and ``delete_worker`` deletes the
    row without touching config, so a host removed and re-enrolled comes back
    under a new id — silently orphaning its watts and dedicated settings while
    the old keys linger. ``client_id`` is UNIQUE and survives re-enrolment,
    which is what these settings should have been keyed on.
    """
    return str(worker.get("client_id") or worker.get("id") or "")


# ---------------------------------------------------------------------------
# API: User Preferences (onboarding state)
# ---------------------------------------------------------------------------


@app.get("/api/preferences")
async def api_get_preferences(request: Request) -> dict[str, Any]:
    user = _require_auth_api(request)
    prefs = await database.get_user_preferences(user["uid"])
    if not prefs:
        return {
            "setup_mode": "fresh",
            "selected_categories": "[]",
            "timezone": "UTC",
            "setup_completed": False,
            "theme": None,
        }
    return prefs


THEMES = database.THEMES


class PreferencesUpdate(BaseModel):
    setup_mode: str | None = None
    selected_categories: str | None = None
    timezone: str | None = None
    setup_completed: bool | None = None
    theme: str | None = None


@app.post("/api/preferences")
async def api_set_preferences(
    request: Request, body: PreferencesUpdate, user: dict[str, Any] = Depends(_require_auth_api)
) -> dict[str, str]:
    if body.setup_mode is not None and body.setup_mode not in ("fresh", "monitoring", "mixed"):
        raise HTTPException(status_code=400, detail="setup_mode must be fresh, monitoring, or mixed")
    if body.theme is not None and body.theme not in THEMES:
        raise HTTPException(status_code=400, detail=f"theme must be one of: {', '.join(THEMES)}")

    # Only the fields that were sent are written (one upsert, no read first),
    # so a theme click racing the wizard's POST cannot undo either.
    await database.save_user_preferences(
        user["uid"],
        setup_mode=body.setup_mode,
        selected_categories=body.selected_categories,
        timezone=body.timezone,
        setup_completed=body.setup_completed,
        theme=body.theme,
    )
    # Saving the preference is a viewer-safe act; triggering a fleet-wide
    # collection is not. /api/collect gates the identical call behind
    # _require_writer, so this was the same side effect through a weaker door —
    # a viewer could hit every provider API on demand.
    #
    # The preference itself still saves for any authenticated user; only the
    # collection is gated, so a viewer completing setup is not blocked.
    if body.setup_completed and auth.require_role(user, "owner", "writer"):
        _spawn(_run_collection())
    return {"status": "saved"}


# ---------------------------------------------------------------------------
# API: Environment Info
# ---------------------------------------------------------------------------


# --- Self-update surface (issue #342) ------------------------------------
#
# The daily release check (app/update_check.py, CashPilot-w0ss) already knows
# the newest published release; these routes add what an operator can DO about
# it. `behind` from update_check.state() is deliberately series-based; the
# update button's semantic is different — it installs the newest PATCH of the
# series the compose file pins — so patch_available is computed separately.


@app.get("/api/update-info")
async def api_update_info(request: Request, worker_id: int | None = None) -> dict[str, Any]:
    """Everything the settings card needs: versions + what an update would do."""
    _require_owner(request)
    wid = await _resolve_worker_id(worker_id)
    info = await _proxy_to_worker(wid, "GET", "/api/update/info")
    check = update_check.state()
    current = version.current()
    latest = check.get("latest")
    same_series = version.series(latest) is not None and version.series(latest) == version.series(current)
    info.update(
        {
            "current": current,
            "current_display": version.display(),
            "latest": latest,
            "known": check.get("known", False),
            "behind": check.get("behind", False),
            # True only when the documented `compose pull` would actually land a
            # newer image on a series-pinned install.
            "patch_available": same_series and version.is_newer(latest, current),
        }
    )
    return info


@app.post("/api/update")
async def api_update(request: Request, worker_id: int | None = None) -> dict[str, Any]:
    """Run the documented update on the selected worker's install.

    The worker refuses with instructions (409) when the install is not
    compose-managed; that detail passes through to the caller verbatim. The
    long timeout covers the helper-image pull on a slow line.
    """
    _require_owner(request)
    wid = await _resolve_worker_id(worker_id)
    return await _proxy_to_worker(wid, "POST", "/api/update/self", timeout=120)


@app.get("/api/env-info")
async def api_env_info(request: Request) -> list[dict[str, Any]]:
    _require_owner(request)
    # (key, label, secret, read_only, default, description)
    env_defs = [
        ("CASHPILOT_API_KEY", "Fleet API Key", True, False, "", "Bearer token for worker-to-UI authentication"),
        (
            "CASHPILOT_SECRET_KEY",
            "Session Secret Key",
            True,
            False,
            "changeme-generate-a-random-secret",
            "Secret for JWT session tokens — change from default for security",
        ),
        (
            "CASHPILOT_HOSTNAME_PREFIX",
            "Hostname Prefix",
            False,
            False,
            "cashpilot",
            "Containers named {prefix}-{service}",
        ),
        (
            "CASHPILOT_COLLECT_INTERVAL",
            "Collect Interval (min)",
            False,
            False,
            "60",
            "Minutes between automatic earnings collection",
        ),
        ("CASHPILOT_DATA_DIR", "Data Directory", False, True, "/data", "Directory containing the SQLite database"),
        ("TZ", "System Timezone", False, False, "", "Container timezone in IANA format (e.g. Europe/Madrid)"),
    ]
    result = []
    for key, label, secret, read_only, default, desc in env_defs:
        raw = os.getenv(key, "")
        entry: dict[str, Any] = {
            "key": key,
            "label": label,
            "secret": secret,
            "read_only": read_only,
            "description": desc,
            "set_via_env": bool(raw),
        }
        if key == "CASHPILOT_SECRET_KEY":
            # Auth always resolves a key at runtime (env, persisted, or generated),
            # so it is effectively always set; never expose its value and treat as
            # read-only in the UI.
            entry["is_set"] = True
            entry["read_only"] = True
        elif secret:
            # Drop the value for secrets — only report presence.
            entry["is_set"] = bool(raw)
        else:
            entry["value"] = raw or default
        result.append(entry)
    return result


# ---------------------------------------------------------------------------
# API: Collectors Metadata
# ---------------------------------------------------------------------------


@app.get("/api/collectors/meta")
async def api_collectors_meta(request: Request) -> list[dict[str, Any]]:
    _require_owner(request)
    from app.collectors import _COLLECTOR_ARGS, COLLECTOR_MAP

    # Single-sourced from database.SECRET_CONFIG_KEYS so this endpoint can never
    # disagree with the encryption-at-rest / masking logic about which config
    # keys are secret (a hand-maintained duplicate here previously missed
    # `remember_web` and `xsrf_token`, unmasking them).
    secret_args = database.SECRET_CONFIG_KEYS
    # Credential hints live in the service YAML (collector.credential_hint),
    # not here. They are per-service prose about where to find a token in a
    # provider's UI, which is exactly the service-specific knowledge the
    # catalog exists to hold — 'never hardcode service-specific logic in
    # app/'. Kept in app/ they also drifted out of reach of anyone editing
    # the service they describe.
    meta = []
    for slug in sorted(COLLECTOR_MAP.keys()):
        args = _COLLECTOR_ARGS.get(slug, [])
        svc = catalog.get_service(slug)
        name = svc.get("name", slug) if svc else slug
        fields = []
        for arg in args:
            optional = arg.startswith("?")
            arg_name = arg.lstrip("?")
            config_key = f"{slug}_{arg_name}"
            fields.append(
                {
                    "key": config_key,
                    "label": arg_name.replace("_", " ").title(),
                    "secret": arg_name in secret_args,
                    "required": not optional,
                }
            )
        # Payment currency for bonus offset labeling
        payment = (svc.get("payment", {}) if svc else {}) or {}
        pay_currency = payment.get("currency", "USD")

        entry: dict[str, Any] = {"slug": slug, "name": name, "fields": fields, "currency": pay_currency}
        hint = (svc.get("collector") or {}).get("credential_hint") if svc else None
        if hint:
            entry["hint"] = hint
        meta.append(entry)
    return meta


# ---------------------------------------------------------------------------
# API: Config
# ---------------------------------------------------------------------------


@app.get("/api/config")
async def api_get_config(request: Request) -> dict[str, Any]:
    _require_owner(request)
    # Masked read path: non-secret values plus a {secret_key: is_set} map under
    # "_secrets". Stored credentials never cross the wire in plaintext.
    return await database.get_config_masked()


class ConfigUpdate(BaseModel):
    data: dict[str, str]


def _sanitize_credential(value: str) -> str:
    """Clean common copy-paste artifacts from credential values."""
    from urllib.parse import unquote

    v = value.strip()
    if v.startswith('"') and v.endswith('"'):
        v = v[1:-1]
    if v.startswith("'") and v.endswith("'"):
        v = v[1:-1]
    if "%3D" in v or "%3d" in v or "%2F" in v or "%2f" in v or "%2B" in v or "%2b" in v:
        v = unquote(v)
    return v


@app.post("/api/config")
async def api_set_config(
    request: Request, body: ConfigUpdate, _auth: dict[str, Any] = Depends(_require_owner)
) -> dict[str, str]:
    sanitized = {k: _sanitize_credential(v) for k, v in body.data.items()}
    before = await database.get_config() or {}
    await database.set_config_bulk(sanitized)
    # Settings posts every prefilled field on the page, so a removed service's
    # unchanged email would bring it back as "balance only". A value that
    # changed counts as the user adding it again, and so does any secret: the
    # page never prefills those, so one in the request was typed.

    # Auto-create "external" deployment records for manual-only services
    # whose collector credentials were just saved.  Without a deployment
    # row, _run_collection() will never instantiate the collector.
    from app.collectors import fully_configured_slugs

    # Completeness is judged against the MERGED config, not this request.
    #
    # set_config_bulk UPSERTS, so a credential set can legitimately arrive
    # across several requests — email now, password a moment later. Checking
    # only `sanitized` meant neither request ever saw a complete set, so both
    # values landed in the database and no deployment row was ever created:
    # credentials stored, collection still dead, and nothing to indicate why.
    #
    # Still scoped to the slugs this request touched, so saving one service's
    # credentials does not sweep the whole catalog.
    #
    # The completeness rule itself lives in collectors.fully_configured_slugs,
    # shared with the startup backfill so the two can never disagree about which
    # services are ready to collect.
    stored = await database.get_config() or {}
    removed = {k[len(database._REMOVED_PREFIX) :] for k in stored if k.startswith(database._REMOVED_PREFIX)}
    for slug in fully_configured_slugs(stored):
        if not any(k.startswith(f"{slug}_") for k in sanitized):
            continue
        if slug in removed and not any(
            k.startswith(f"{slug}_") and v and (v != before.get(k) or database._is_secret_key(k))
            for k, v in sanitized.items()
        ):
            continue
        svc = catalog.get_service(slug)
        if not svc:
            continue
        # No has_image gate. Collection is driven by DEPLOYMENT ROWS
        # (make_collectors iterates deployments), so a service with no row is
        # never collected no matter how complete its credentials are. Skipping
        # image-backed services here meant that for 12 of the 15 collectors,
        # saving credentials did nothing at all — while Settings promised
        # "You don't need to deploy containers through CashPilot — just add
        # your credentials." The badge flipped to Configured and no earnings
        # ever arrived.
        #
        # Creating the row is safe for a user who later deploys through
        # CashPilot: api_deploy calls save_deployment with the real container
        # id and status, replacing this placeholder.
        existing = await database.get_deployment(slug)
        if not existing:
            await database.save_deployment(slug=slug, container_id="", status="external")
            logger.info("Tracking %s from stored credentials (no container deployed by CashPilot)", slug)

    return {"status": "saved"}


@app.delete("/api/config/{slug}")
async def api_clear_service_config(request: Request, slug: str) -> dict[str, str]:
    """Remove all stored credentials (and signup bonus) for a service."""
    _require_owner(request)
    from app.collectors import _COLLECTOR_ARGS

    arg_keys = _COLLECTOR_ARGS.get(slug)
    if not arg_keys:
        raise HTTPException(status_code=404, detail="Unknown service")

    config_keys = [f"{slug}_{a.lstrip('?')}" for a in arg_keys]
    config_keys.append(f"{slug}_signup_bonus")
    await database.delete_config_keys(config_keys)

    # Remove the row only if it is the placeholder we auto-created.
    #
    # Gated on status == "external", NOT on whether the service has an image.
    # Clearing credentials must never undeploy a container the user actually
    # deployed through CashPilot — that row has a real container id and a
    # status of its own, and removing it would orphan a running container from
    # the dashboard.
    existing = await database.get_deployment(slug)
    if existing and (existing.get("status") or "") == "external":
        await database.remove_deployment(slug)

    logger.info("Cleared credentials for %s", slug)
    return {"status": "cleared"}


# ---------------------------------------------------------------------------
# API: Users — change password (owner-reset + self-service).
#
# User list/role/delete routes live in app.routers.users. The password routes
# stay here to avoid the direct-import problem (no test imports them directly).
# ---------------------------------------------------------------------------


class PasswordChange(BaseModel):
    current_password: str
    new_password: str


class AdminPasswordSet(BaseModel):
    new_password: str


@app.post("/api/users/me/password")
async def api_change_own_password(
    request: Request, body: PasswordChange, user: dict[str, Any] = Depends(_require_auth_api)
) -> JSONResponse:
    """Change the authenticated user's own password (verifies current password)."""
    uid = user["uid"]
    if uid == 0:
        raise HTTPException(status_code=400, detail="API-key sessions cannot change a password")
    record = await database.get_user_by_id(uid)
    if not record:
        raise HTTPException(status_code=404, detail="User not found")
    if not await auth.verify_password_async(body.current_password, record["password"]):
        raise HTTPException(status_code=403, detail="Current password is incorrect")
    if len(body.new_password) < 10:
        raise HTTPException(status_code=400, detail="Password must be at least 10 characters")
    if body.new_password == body.current_password:
        raise HTTPException(status_code=400, detail="New password must differ from the current password")

    hashed = await auth.hash_password_async(body.new_password)
    await database.update_user_password(uid, hashed)
    changed = await database.get_user_by_id(uid)
    auth.set_user_pwd_epoch(uid, changed["password_changed_at"])
    # Re-mint the session cookie so the caller stays logged in after the epoch bump.
    token = auth.create_session_token(uid, user["u"], user["r"])
    return auth.set_session_cookie(JSONResponse({"status": "password_changed"}), token, request)


@app.post("/api/users/{user_id}/password")
async def api_admin_set_password(
    request: Request, user_id: int, body: AdminPasswordSet, _auth: dict[str, Any] = Depends(_require_owner)
) -> dict[str, str]:
    """Owner resets another user's password (no current-password check, no re-mint)."""
    target = await database.get_user_by_id(user_id)
    if not target:
        raise HTTPException(status_code=404, detail="User not found")
    if len(body.new_password) < 10:
        raise HTTPException(status_code=400, detail="Password must be at least 10 characters")
    hashed = await auth.hash_password_async(body.new_password)
    await database.update_user_password(user_id, hashed)
    changed = await database.get_user_by_id(user_id)
    auth.set_user_pwd_epoch(user_id, changed["password_changed_at"])
    return {"status": "password_set"}


# ---------------------------------------------------------------------------
# API: Fleet (Workers)
# ---------------------------------------------------------------------------


def _bearer_token(request: Request) -> str:
    """Extract the bearer token from an Authorization header (empty if absent)."""
    h = request.headers.get("Authorization", "")
    return h[7:] if h.startswith("Bearer ") else ""


WORKER_KEY_CONFIRM_WINDOW = timedelta(hours=24)


def _humanize_window() -> str:
    hours = WORKER_KEY_CONFIRM_WINDOW.total_seconds() / 3600
    return f"more than {hours:g} hours" if hours >= 1 else f"more than {WORKER_KEY_CONFIRM_WINDOW.total_seconds():g}s"


def enrollment_state(key_issued_at: str | None, confirmed: bool, now: datetime | None = None) -> str:
    """``"confirmed"``, ``"pending"`` (still inside the window) or ``"incomplete"``.

    Pure, so the heartbeat guard and the fleet page cannot disagree about which
    workers are still enrolling and which have stalled.

    An UNPARSEABLE or missing timestamp reads as ``"pending"``. That is the safe
    direction: reading unknown as expired would lock out a worker on the
    strength of a value nobody wrote.
    """
    if confirmed:
        return "confirmed"
    if not key_issued_at:
        return "pending"
    try:
        issued = datetime.fromisoformat(key_issued_at).replace(tzinfo=UTC)
    except (TypeError, ValueError):
        logger.warning(
            "Worker key issue time %r is unparseable; treating enrollment as still in progress", key_issued_at
        )
        return "pending"
    return "pending" if (now or datetime.now(UTC)) - issued <= WORKER_KEY_CONFIRM_WINDOW else "incomplete"


async def _enrollment_window_open(cid: str) -> bool:
    """Whether the shared key may still be honoured for this enrolled worker.

    A second read of the same row, rather than widening ``get_worker_key_state``.
    It is reached only on the rare branch — a worker that has a key, has never
    used it, and is presenting the shared key. Every steady-state heartbeat
    returns "ok" before this line, so this is not on the hot path.
    """
    return enrollment_state(await database.get_worker_key_issued_at(cid), confirmed=False) == "pending"


async def _authenticate_worker_heartbeat(request: Request, cid: str) -> str:
    """Authenticate a heartbeat and classify it. Returns one of:

    - ``"enroll"``  — worker has no key yet and presented the shared key: mint one.
    - ``"reissue"`` — worker has a key that is NOT yet confirmed and presented the
      shared key: it likely lost the enrollment response, so re-deliver the SAME
      key. Until confirmed, the shared key still works for this one worker — a
      window that closes on the worker's first own-key heartbeat OR after
      ``WORKER_KEY_CONFIRM_WINDOW`` from the key being minted, whichever comes
      first.

    That second bound is the whole point of this function. The window used to
    close only on confirmation, so a worker that CANNOT persist its key — a
    pre-1.0.0 image, a read-only /data, an ephemeral container — never closed it
    at all. The security cutover the per-worker keys exist for silently never
    completed: anyone holding CASHPILOT_API_KEY could impersonate that worker
    indefinitely, and the UI re-sent the key to them every 60 seconds while
    logging "not yet confirmed" once a minute forever.
    - ``"ok"``      — worker presented its own key: authenticated; confirm it so the
      shared key is refused from now on (the cutover finalizes).

    Raises 401 otherwise — notably, a confirmed worker presenting the shared key is
    rejected, which is what stops shared-key holders from impersonating a worker.
    """
    if not FLEET_API_KEY:
        raise HTTPException(
            status_code=503,
            detail="Fleet key not configured — set CASHPILOT_API_KEY or mount shared /fleet volume",
        )
    token = _bearer_token(request)
    key, confirmed = await database.get_worker_key_state(cid) if cid else (None, False)
    shared_ok = bool(token) and hmac.compare_digest(token.encode(), FLEET_API_KEY.encode())
    if key is None:
        if shared_ok:
            return "enroll"
        raise HTTPException(status_code=401, detail="Invalid API key")
    if token and hmac.compare_digest(token.encode(), key.encode()):
        return "ok"
    if not confirmed and shared_ok:
        if await _enrollment_window_open(cid):
            return "reissue"
        # Past the window. Refusing here is the cutover finally completing for a
        # worker that could never complete it itself. Said plainly, because the
        # operator's next question is why a worker that was working stopped.
        logger.warning(
            "Worker '%s' presented the SHARED key %s after enrolling and has never used its own key. "
            "The shared key is no longer accepted for it: a worker that cannot persist "
            "/data/.worker_key would otherwise keep the shared key valid for its identity forever. "
            "Upgrade that worker to 1.0.0+, give it a writable and PERSISTENT /data, then remove it "
            "in the fleet page so it can enroll again.",
            cid,
            _humanize_window(),
        )
        raise HTTPException(
            status_code=401,
            detail=(
                "Enrollment was never completed for this worker and the shared key is no longer "
                "accepted for it. Upgrade the worker to 1.0.0+ with a writable, persistent /data, "
                "then remove it in the fleet page so it can enroll again."
            ),
        )
    raise HTTPException(status_code=401, detail="Invalid or missing per-worker key")


class WorkerHeartbeat(BaseModel):
    name: str
    url: str = ""
    client_id: str = ""
    containers: list[dict[str, Any]] = []
    apps: list[dict[str, Any]] = []
    system_info: dict[str, Any] = {}
    # One Activity window (app/activity.py), sent only by a worker the owner
    # switched on. Declared, or pydantic would drop it silently.
    activity: dict[str, Any] | None = None


def _android_app_status(running: object) -> str:
    """How an Android app's three-valued ``running`` reads as a container status.

    ``None`` is UNKNOWN and must not become ``"stopped"``. The phone reports it
    when it cannot see -- notification-listener and usage access are what make
    detection possible at all, and without them every signal reads false. A
    worker on an older client that never sends null is unaffected.
    """
    if running is None:
        return "unknown"
    return "running" if running else "stopped"


async def _earnings_for_worker(body: WorkerHeartbeat, days: int = 30) -> dict[str, Any] | None:
    """What the platforms on THIS worker have earned, for the client to display.

    Returned in the heartbeat response rather than from a new endpoint, because
    the heartbeat is the ONE call a worker is already authenticated for. Every
    earnings route goes through ``_require_auth_api``, which needs a user
    session; a per-worker key cannot read any of them, and handing a phone an
    owner-level credential so it can show a number would be a bad trade.

    THE HONESTY CONSTRAINT, which shapes the whole payload:

    Earnings are collected per PLATFORM, from the provider's account. They are
    not, and cannot be, attributed to a device. If two machines both run Grass,
    the provider reports one balance and nothing can split it. So this never
    says "this device earned X" — it says "the platforms this device is running
    earned X on your account", and it flags each platform that is running on
    more than one worker so the client can say so too.

    Returns None when the figures cannot be produced at all. None means unknown;
    a caller must not render it as zero.
    """
    slugs = {
        str(entry.get("slug") or "").strip()
        for entry in (body.apps or []) + (body.containers or [])
        if str(entry.get("slug") or "").strip()
    }
    if not slugs:
        return None

    try:
        earned = await database.get_earned_by_platform(days)
        workers = await database.list_workers()
    except Exception as exc:  # noqa: BLE001 - a heartbeat must never fail on this
        logger.warning("Could not attach earnings to the heartbeat response: %s", exc)
        return None

    # Which platforms run on more than one worker. Counted across the fleet, not
    # just this device, because that is what makes a per-device claim false.
    running_on: dict[str, int] = {}
    for worker in workers:
        _parse_worker_json(worker)
        seen: set[str] = set()
        for entry in (worker.get("apps") or []) + (worker.get("containers") or []):
            slug = str(entry.get("slug") or "").strip()
            if slug and slug not in seen:
                seen.add(slug)
                running_on[slug] = running_on.get(slug, 0) + 1

    platforms = []
    for slug in sorted(slugs):
        # A platform with no reading is UNKNOWN, not zero. Absent from `earned`
        # means nothing has ever been collected for it -- most often no collector
        # exists, or its credentials were never entered.
        value = earned.get(slug)
        platforms.append(
            {
                "slug": slug,
                "usd": round(value, 4) if value is not None else None,
                "shared_with_other_workers": running_on.get(slug, 0) > 1,
            }
        )

    known = [p["usd"] for p in platforms if p["usd"] is not None]
    return {
        "window_days": max(1, int(days)),
        "currency": "USD",
        "platforms": platforms,
        # Only sums what is actually known. A total that silently treats unknown
        # as zero is the same lie in aggregate.
        "total_usd": round(sum(known), 4) if known else None,
        "platforms_without_readings": [p["slug"] for p in platforms if p["usd"] is None],
    }


#: The exact shape ``upsert_earnings`` writes and both delta readers ORDER BY.
#: A date in any other shape would sort into the wrong place in its own series,
#: so the readings either side of it difference against the wrong neighbour --
#: silently, and only for the client that sent it.
_ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")


class EarningsReading(BaseModel):
    """One historical balance reading from a paired client."""

    slug: str
    #: allow_inf_nan=False because JSON's `NaN` and `Infinity` -- which Python's
    #: parser accepts even though the spec does not -- are otherwise stored
    #: verbatim. One NaN balance poisons every delta taken from that series
    #: (NaN - x is NaN, and every comparison against it is False, so the clamp
    #: silently misbehaves), the account total becomes NaN, and serialising that
    #: back out emits a bare `NaN` that JSON.parse rejects. So a single bad
    #: reading from one client breaks the dashboard for everyone.
    balance: float = Field(allow_inf_nan=False)
    date: str
    currency: str = "USD"
    fx_rate_usd: float | None = Field(default=None, allow_inf_nan=False)

    @field_validator("date")
    @classmethod
    def _iso_date(cls, value: str) -> str:
        value = (value or "").strip()
        if not _ISO_DATE.match(value):
            raise ValueError("date must be YYYY-MM-DD")
        # Reject 2026-02-30 and friends: the pattern above only proves shape.
        try:
            datetime.strptime(value, "%Y-%m-%d")  # noqa: DTZ007 -- a calendar date, not an instant
        except ValueError as exc:
            raise ValueError("date must be a real calendar date") from exc
        return value


class EarningsImport(BaseModel):
    """A client pushing the history it collected before it was paired.

    Deliberately carries NO source field. The source is taken from the
    AUTHENTICATED worker, never from the body -- otherwise any enrolled client
    could write into the 'server' series, or into another machine's, and
    overwrite readings it never took.
    """

    client_id: str
    #: Bounded so one authenticated client cannot hand the server an
    #: arbitrarily large body to parse and then write row by row. 2000 is
    #: comfortably above a real import -- the server keeps 400 days, and a
    #: client chunks anything larger -- while staying a body the process can
    #: hold. An unbounded list here is a denial of service that needs only one
    #: compromised worker.
    readings: list[EarningsReading] = Field(default_factory=list, max_length=2000)


@app.post("/api/workers/earnings-import")
async def api_worker_earnings_import(request: Request, body: EarningsImport) -> dict[str, Any]:
    """Accept a paired client's historical earnings under its own source.

    This exists because the server and a Desktop can both have been reading the
    SAME provider account. Their readings must not be merged into one series:
    earnings are clamped deltas between consecutive readings, so interleaving two
    samplers makes every drop clamp to zero and the total comes out
    systematically understated. Each client's rows therefore land under its own
    ``source`` and are differenced separately.

    Idempotent by construction: the (platform, source, date) unique index means a
    re-pair or a retried import UPDATES a day rather than adding a second reading
    for it, which would difference against itself and read as zero.
    """
    cid = (body.client_id or "").strip()
    if not cid:
        raise HTTPException(status_code=400, detail="client_id required")

    state = await _authenticate_worker_heartbeat(request, cid)
    # Only a CONFIRMED worker may import. "enroll" and "reissue" both mean the
    # caller presented the SHARED key, which every worker holds -- accepting it
    # here would let anyone with that token write a history for any client_id
    # they cared to name. A heartbeat is idempotent status; this is durable
    # money data, so it gets the stricter bar.
    if state != "ok":
        raise HTTPException(
            status_code=403,
            detail=(
                "Importing earnings requires this worker's own key. Send a heartbeat first "
                "to complete enrollment, then retry."
            ),
        )

    known = {s["slug"] for s in catalog.get_services()}
    # DISTINCT slugs, not one entry per skipped reading. A client pushing 400
    # days of a platform this server does not know would otherwise get the same
    # name back 400 times -- a response that grows with the request, echoing
    # client-supplied strings, and tells the reader nothing the set does not.
    skipped: set[str] = set()
    rows: list[dict[str, Any]] = []
    for reading in body.readings:
        slug = (reading.slug or "").strip()
        # An unknown slug is dropped rather than stored: it would create a
        # platform the catalog cannot render, name or ever collect for again.
        if not slug or slug not in known:
            skipped.add(slug or "(blank)")
            continue
        rows.append(
            {
                "platform": slug,
                "balance": float(reading.balance),
                "currency": (reading.currency or "USD").upper(),
                "date": reading.date,
                "fx_rate_usd": reading.fx_rate_usd,
                "source": cid,
            }
        )

    # ONE transaction for the whole batch. Writing row by row committed once per
    # reading, and every commit is an fsync that takes SQLite's write lock -- so
    # a thousand-reading import serialised a thousand disk syncs against this
    # server's own collector, and latency tracked sync cost rather than row
    # count. It also makes the import all-or-nothing: a failure part-way leaves
    # the client's history exactly as it was rather than half-applied, and the
    # import is idempotent so retrying costs a round trip. (CodeRabbit, PR #256.)
    written = await database.upsert_earnings_many(rows)

    logger.info(
        "Imported %d earnings reading(s) from worker '%s' (%d unknown platform(s) skipped)",
        written,
        cid,
        len(skipped),
    )
    return {"status": "ok", "imported": written, "skipped": sorted(skipped), "source": cid}


@app.post("/api/workers/heartbeat")
async def api_worker_heartbeat(request: Request, body: WorkerHeartbeat) -> dict[str, Any]:
    """Receive a heartbeat from a worker. Registers or updates the worker."""
    # Use client_id for identity; fall back to name for backward compat
    cid = body.client_id or body.name
    if not cid:
        raise HTTPException(status_code=400, detail="Worker name or client_id required")
    state = await _authenticate_worker_heartbeat(request, cid)
    # Pre-upsert state: upsert_worker unconditionally writes 'online', so the
    # offline -> online recovery is only observable here, before it runs.
    # Best-effort on purpose: this read failing must never reject a heartbeat
    # (the upsert below is the loud path if the DB is really broken); the cost
    # is one missed immediate alert-clear, which the 24h cooldown self-heals.
    previous = None
    try:
        previous = await database.get_worker_status_and_name(cid)
    except Exception as exc:  # noqa: BLE001
        logger.debug("Could not read pre-heartbeat worker state for %s: %s", cid, exc)
    worker_id = await database.upsert_worker(
        client_id=cid,
        name=body.name,
        url=body.url,
        containers=json.dumps(body.containers),
        apps=json.dumps(body.apps),
        system_info=json.dumps(body.system_info),
    )
    metrics.record_heartbeat(body.name)
    # Hourly CPU, memory and traffic per container for the dashboard's usage
    # graphs. Never allowed to reject a heartbeat: losing one sample costs a
    # minute of data.
    if not _is_phone(body.system_info) and body.containers:
        try:
            await database.record_usage(worker_id, body.containers)
        except Exception as exc:  # noqa: BLE001
            logger.warning("Could not record resource usage for worker %s: %s", worker_id, exc)
    # Traffic through each proxy, for its cap. Never allowed to reject a
    # heartbeat: a missed reading is one minute of uncounted traffic.
    proxied = (body.system_info or {}).get("proxied") if isinstance(body.system_info, dict) else None
    if isinstance(proxied, list) and proxied:
        try:
            await _account_proxy_traffic(worker_id, proxied)
        except Exception as exc:  # noqa: BLE001
            logger.warning("Could not count proxy traffic for worker %s: %s", worker_id, exc)
    # Activity: the switch is read on every heartbeat, because the reply is
    # how the worker learns it. Never allowed to reject a heartbeat, and the
    # log line never carries the exception text, which could quote an address.
    # ``stored`` tells the worker it may drop the window; it is False only when
    # the write failed, so the worker sends the window again.
    activity_on: bool | None = None
    activity_stored: bool | None = None
    if not _is_phone(body.system_info):
        try:
            activity_on = activity_ui.enabled(await database.get_config() or {}, {"client_id": cid, "id": worker_id})
            if activity_on and body.activity is not None:
                activity_stored = False
                await _ingest_activity(worker_id, cid, body.name, body.activity)
                activity_stored = True
        except Exception as exc:  # noqa: BLE001
            logger.warning("Could not record activity for worker %s (%s)", worker_id, type(exc).__name__)
    if previous is not None and previous[0] == "offline":
        # Deliberately NO alert clearing here. This used to drop the stored
        # alert on the first heartbeat after offline — which re-armed the push
        # on every single beat, so a phone worker waking from Android Doze
        # every ~12 minutes produced an identical offline push per nap (seven
        # in 80 minutes, live). The stale-worker sweep now owns clearing, and
        # only after the worker stays online for _SUSTAINED_RECOVERY_SWEEPS
        # consecutive sweeps — a real recovery re-arms in ~4 minutes, a Doze
        # blip never does, and the 24h record cooldown keeps one push per
        # episode.
        logger.info("Worker '%s' is back online", previous[1])
    resp: dict[str, Any] = {"status": "ok", "worker_id": worker_id}
    if state == "enroll":
        # First contact: mint this worker's own key and hand it back once. Stored
        # unconfirmed until the worker proves receipt by using it (see "ok").
        new_key = secrets.token_urlsafe(32)
        await database.set_worker_key(cid, new_key)
        resp["worker_key"] = new_key
        logger.info("Worker '%s' enrolled — per-worker key issued (awaiting confirmation)", cid)
    elif state == "reissue":
        # The worker still holds the shared key, so it never received/persisted its
        # own key — re-deliver the same one so a dropped enrollment response can't
        # lock it out.
        existing = await database.get_worker_key(cid)
        if existing:
            resp["worker_key"] = existing
        logger.info("Worker '%s' re-issued its per-worker key (not yet confirmed)", cid)
    elif state == "ok":
        # The worker authenticated with its own key: finalize the cutover so the
        # shared key is refused from now on.
        await database.confirm_worker_key(cid)

    # What the platforms on this worker have earned, so a client can show money
    # without needing a second, broader credential. Absent when it cannot be
    # produced -- the key is omitted rather than sent as an empty object, so a
    # client can tell "unknown" from "nothing earned" (CashPilot-android-35t).
    earnings = await _earnings_for_worker(body)
    if earnings is not None:
        resp["earnings"] = earnings
    if activity_on is not None:
        resp["activity"] = {"enabled": activity_on}
        if activity_stored is not None:
            resp["activity"]["stored"] = activity_stored
    return resp


async def _ingest_activity(worker_id: int, cid: str, name: str, payload: dict[str, Any]) -> None:
    """Store one Activity window and raise its alerts. Raises on a database failure."""
    if len(json.dumps(payload, separators=(",", ":"))) > activity_ui.MAX_PAYLOAD_BYTES:
        logger.warning("Activity window from worker %s ignored: over the size limit", worker_id)
        return
    window = activity_ui.sanitise(payload, time.time())
    if window is None:
        logger.warning("Activity window from worker %s ignored: not a version this UI reads", worker_id)
        return
    activity_ui.LAST_STATUS[worker_id] = {
        "worker_status": window["worker_status"],
        "leftover_probes": window["leftover_probes"],
        "window_end": window["window_end"],
    }
    public = [d["ip"] for s in window["samples"] for d in s["out"] + s["in"] if d["class"] == "public"]
    geo = await asyncio.to_thread(ipgeo.lookup, public) if public else {}
    keys = (activity_ui.flag_key({"client_id": cid, "id": worker_id}), f"worker_{worker_id}_{activity_ui.FLAG_SUFFIX}")
    if not await database.record_activity(worker_id, window, geo, flag_keys=keys):
        # Nothing stored: the switch went off (or the box was removed) after the
        # heartbeat read it, or this window was stored already. Either way its
        # alerts are not raised again.
        return
    raised = False
    for f in activity_ui.alert_findings(window):
        subject = f"{cid}:{f['container']}" + (f":{f['detail']}" if f["detail"] else "")
        cooldown = activity_ui.MAIL_COOLDOWN_HOURS if f["kind"] == "activity_mail" else activity_ui.LAN_COOLDOWN_HOURS
        message = f"On {name}: {f['message']}"
        if alert_id := await database.record_alert(f["kind"], subject, message, cooldown_hours=cooldown):
            raised = True
            what = "sent mail" if f["kind"] == "activity_mail" else "reached into your network"
            _spawn(_push_alert(f["kind"], subject, f"CashPilot: {f['container']} {what}", message, alert_id))
    if raised:
        await _refresh_activity_bell()


async def _refresh_activity_bell() -> None:
    """Put the last day's Activity alerts in the bell, one per subject."""
    global _activity_bell, _collector_alerts
    try:
        stored = await database.list_alerts(limit=500)
    except Exception as exc:  # noqa: BLE001
        logger.warning("Could not read activity alerts: %s", type(exc).__name__)
        return
    cutoff = (datetime.now(UTC) - timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S")
    seen: set[str] = set()
    bell = []
    for a in stored:
        if not str(a.get("kind", "")).startswith("activity_") or str(a.get("created_at", "")) < cutoff:
            continue
        if a["subject"] in seen:
            continue
        seen.add(a["subject"])
        parts = a["subject"].split(":")
        bell.append(
            {"kind": a["kind"], "platform": parts[1] if len(parts) > 1 else a["subject"], "error": a["message"]}
        )
    _activity_bell = bell
    _collector_alerts = [a for a in _collector_alerts if not str(a.get("kind", "")).startswith("activity_")] + bell


async def _activity_spikes(workers: list[dict[str, Any]], now: float | None = None) -> None:
    """Bell-only alerts for an hour far above a container's usual one.

    Bytes come from usage_hourly (no second byte counter), connections from
    the Activity windows. Each needs SPIKE_MIN_HOURS of history; unknown hours
    are skipped, never read as zero.
    """
    now = time.time() if now is None else now
    last_hour = int(now // 3600) * 3600 - 3600
    since = last_hour - 7 * 86400
    opens = await database.get_activity_opens_hourly(since)
    for w in workers:
        cid = str(w.get("client_id") or w["id"])
        per: dict[tuple[str, str], dict[int, float | None]] = {}
        for r in await database.get_usage_hourly(since, w["id"]):
            per.setdefault((r["container"], "received"), {})[r["hour"]] = r["rx_bytes"]
            per.setdefault((r["container"], "sent"), {})[r["hour"]] = r["tx_bytes"]
        for r in opens:
            if r["worker_id"] == w["id"]:
                per.setdefault((r["container"], "opens"), {})[r["hour"]] = r["opens"]
        for (container, what), hours in per.items():
            latest = hours.get(last_hour)
            history = [v for h, v in hours.items() if h < last_hour]
            floor = activity_ui.SPIKE_OPENS_FLOOR if what == "opens" else activity_ui.SPIKE_BYTES_FLOOR
            if not activity_ui.spike(history, latest, floor):
                continue
            if what == "opens":
                detail = f"opened {int(latest or 0)} connections in the last hour"
            else:
                detail = f"{what} {activity_ui.human_bytes(latest or 0)} in the last hour"
            message = (
                f"On {w.get('name') or cid}: {container} {detail}, more than {activity_ui.SPIKE_RATIO} times "
                "its usual hour this week."
            )
            await database.record_alert(
                "activity_spike",
                f"{cid}:{container}:{'opens' if what == 'opens' else 'bytes'}",
                message,
                cooldown_hours=activity_ui.SPIKE_COOLDOWN_HOURS,
            )


async def _run_activity_purge() -> None:
    """Purge raw addresses past a day and the rows of removed boxes. Never raises."""
    try:
        purged = await database.purge_activity_raw()
        if purged:
            logger.info("Activity: purged %d rows (raw addresses past a day, removed boxes)", purged)
    except Exception as exc:  # noqa: BLE001
        logger.warning("Activity purge failed: %s", type(exc).__name__)


async def _run_activity_hourly() -> None:
    """Purge raw addresses, check for spikes, refresh the network table when due."""
    await _run_activity_purge()
    try:
        config = await database.get_config() or {}
        opted = [
            w
            for w in await database.list_workers()
            if not _is_phone(_safe_json(w.get("system_info", "{}"), {})) and activity_ui.enabled(config, w)
        ]
    except Exception as exc:  # noqa: BLE001
        logger.warning("Activity check could not read the workers: %s", type(exc).__name__)
        return
    if opted:
        try:
            await _activity_spikes(opted)
        except Exception as exc:  # noqa: BLE001
            logger.warning("Activity spike check failed: %s", type(exc).__name__)
        if ipgeo.needs_refresh():
            try:
                rows = await asyncio.to_thread(ipgeo.download)
                logger.info("Activity: network table refreshed (%d ranges)", rows)
            except Exception as exc:  # noqa: BLE001
                logger.warning("Activity: network table download failed (%s)", type(exc).__name__)
    await _refresh_activity_bell()


@app.get("/api/workers")
async def api_list_workers(request: Request) -> list[dict[str, Any]]:
    """List all registered workers."""
    _require_auth_api(request)
    workers = await database.list_workers()
    config = await database.get_config() or {}
    ui_version = version.current()
    for w in workers:
        _parse_worker_json(w)
        # Skew is judged here, where the UI's own version is known, rather than
        # in the browser: the fleet page would otherwise have to learn what the
        # UI is running and re-implement the comparison. Both sides must be
        # known releases for this to be True, so an older worker that predates
        # version reporting reads as unknown, not as a mismatch.
        w["ui_version"] = ui_version
        # An Android client ships on its OWN release track, so the UI's version
        # is the wrong yardstick: a phone on 0.3.0 against a UI on 1.24.1 is not
        # skew, it is two different products, and comparing them flagged EVERY
        # Android device permanently. Compare a phone against the newest
        # CashPilot-android release instead.
        #
        # reference_version is what the worker was actually judged against, so
        # the UI can say so rather than always claiming "UI vX".
        si = w.get("system_info") or {}
        is_android = placement.is_android(si)
        reference = update_check.android_latest() if is_android else ui_version
        w["reference_version"] = reference
        # Unknown reference -> no skew. version.skewed() already returns False
        # when either side is unknown, which is what keeps an offline install
        # (or a GitHub outage) from lighting a warning on every phone.
        w["version_skew"] = version.skewed(reference, si.get("version"))
        # The per-machine power settings the running-costs card needs, read the
        # SAME way api_fleet_economics reads them — client_id first, row id as
        # the fallback for values written before that changed. Returning them
        # here is what lets the fleet page show the current value in its input
        # rather than an empty box that silently overwrites what is stored.
        raw_watts = config.get(f"worker_{_worker_config_key(w)}_watts") or config.get(f"worker_{w.get('id')}_watts")
        w["watts"] = raw_watts or ""
        w["dedicated"] = _worker_flag(config, w, "dedicated")
        # Whether this worker ever finished the per-worker-key cutover. Without
        # it the only trace was a UI log line once a minute, which nobody reads,
        # so a worker still authenticating with the SHARED key looked identical
        # to a fully enrolled one on the fleet page.
        w["enrollment"] = enrollment_state(w.get("key_issued_at"), bool(w.get("key_confirmed")))
    return workers


def _parse_worker_json(w: dict[str, Any]) -> None:
    """Parse stored JSON columns and compute counts for a worker dict.

    Also drops the worker's encrypted fleet key, because ``list_workers`` is a
    ``SELECT *`` and every path that renders a worker to a client goes through
    here. The key is a credential: it is what a worker authenticates with, and
    it has no use in a dashboard. Encrypted at rest is not a reason to hand the
    ciphertext to every logged-in viewer — it makes the Fernet key the only
    thing standing between a read-only account and the whole fleet's
    credentials, when nothing needed to publish it at all.

    Stripped here rather than in ``list_workers`` on purpose: the stale-worker
    purge legitimately reads ``api_key_enc`` to tell an enrolled worker from one
    that never enrolled, and it does not go through this function.
    """
    w.pop("api_key_enc", None)
    w["containers"] = _safe_json(w.get("containers", "[]"))
    w["apps"] = _safe_json(w.get("apps", "[]"))
    w["system_info"] = _safe_json(w.get("system_info", "{}"), {})
    is_android = placement.is_android(w["system_info"])
    if is_android:
        w["container_count"] = len(w["apps"])
        w["running_count"] = sum(1 for a in w["apps"] if a.get("running"))
    else:
        w["container_count"] = len(w["containers"])
        w["running_count"] = sum(1 for c in w["containers"] if c.get("status") == "running")


@app.get("/api/workers/{worker_id}")
async def api_get_worker(request: Request, worker_id: int) -> dict[str, Any]:
    """Get details for a specific worker."""
    _require_auth_api(request)
    worker = await database.get_worker(worker_id)
    if not worker:
        raise HTTPException(status_code=404, detail="Worker not found")
    _parse_worker_json(worker)
    return worker


@app.post("/api/workers/{worker_id}/isolation-check")
async def api_worker_isolation_check(request: Request, worker_id: int) -> dict[str, Any]:
    """What an earner on this worker's host can reach (issue #451).

    Owner only: the worker starts two short-lived probe containers on its
    host, one of them on host networking, to answer. POST, because it starts
    containers: the session cookie is SameSite=lax, which still rides a
    cross-site GET navigation.
    """
    _require_owner(request)
    # Two probes of up to 30 s each, plus starting and removing them.
    return await _proxy_to_worker(worker_id, "POST", "/api/isolation-check", timeout=120)


@app.post("/api/workers/{worker_id}/isolation")
async def api_worker_isolation(request: Request, worker_id: int) -> dict[str, Any]:
    """Turn LAN isolation on for an upgraded worker (app/lan_fence.py).

    Owner only, POST for the same reasons as the isolation check: the worker
    creates a network and writes a firewall table on its host. It answers with
    the earners still on Docker's default bridge; the fleet page redeploys them.
    There is no off switch here on purpose: CASHPILOT_ISOLATION=off on the
    worker is the one way out, so a click cannot take the fence down.
    """
    _require_owner(request)
    return await _proxy_to_worker(worker_id, "POST", "/api/isolation", json={"enable": True}, timeout=60)


@app.delete("/api/workers/{worker_id}")
async def api_delete_worker(request: Request, worker_id: int) -> dict[str, str]:
    """Remove a registered worker."""
    _require_owner(request)
    worker = await database.get_worker(worker_id)
    if not worker:
        raise HTTPException(status_code=404, detail="Worker not found")
    await database.delete_worker(worker_id)
    return {"status": "deleted"}


class WorkerCommand(BaseModel):
    command: str  # deploy, stop, restart, start, remove
    slug: str = ""
    spec: dict[str, Any] = {}


@app.post("/api/workers/{worker_id}/command")
async def api_worker_command(request: Request, worker_id: int, body: WorkerCommand) -> dict[str, Any]:
    """Send a command to a worker by proxying to its REST API."""
    # Deploy is owner-gated everywhere else (see /api/deploy/{slug}); a writer
    # must not be able to bypass that gate by sending command="deploy" here.
    if body.command == "deploy":
        _require_owner(request)
    else:
        _require_writer(request)

    if body.command == "deploy":
        # This is a THIRD deploy path. Without the same status gate, a broken or
        # dropped service could still be deployed here — and this route then runs the
        # full bookkeeping below, so it would look deployed while earning nothing.
        # An unknown slug is left alone: this raw route is not catalog-only.
        deploy_status = (catalog.get_service(body.slug) or {}).get("status")
        if deploy_status in _UNDEPLOYABLE_STATUSES:
            raise HTTPException(
                status_code=409 if deploy_status == "broken" else 410,
                detail=f"Service '{body.slug}' is no longer available for deployment ({deploy_status})",
            )
        result = await _proxy_to_worker(worker_id, "POST", f"/api/containers/{body.slug}/deploy", json=body.spec)
    elif body.command in ("stop", "restart", "start"):
        result = await _proxy_to_worker(worker_id, "POST", f"/api/containers/{body.slug}/{body.command}")
    elif body.command == "remove":
        result = await _proxy_to_worker(worker_id, "DELETE", f"/api/containers/{body.slug}")
    else:
        raise HTTPException(status_code=400, detail=f"Unknown command: {body.command}")

    # The primary lifecycle routes (/api/deploy, /api/stop, ...) all run bookkeeping
    # after a successful proxy. Without it a deploy through this raw command route never
    # creates a deployments row — so the service earns $0 forever — and a remove leaks
    # the row. Mirror the canonical routes exactly, keyed on the command, on success only.
    slug = body.slug
    if body.command == "deploy":
        container_id = result.get("container_id", "remote")
        # Record the spec this route actually deployed. It is supplied raw by the
        # caller rather than built from the catalog, so it is the only description
        # of this container that will ever exist.
        await database.save_deployment(
            slug=slug,
            container_id=container_id,
            spec=body.spec if isinstance(body.spec, dict) else None,
            worker_id=worker_id,
        )
        await database.record_health_event(slug, "start", f"deployed to worker {worker_id}")
        metrics.record_container_lifecycle("deploy", slug)
        _spawn(_run_collection())
    elif body.command == "stop":
        await database.record_health_event(slug, "stop")
        metrics.record_container_lifecycle("stop", slug)
    elif body.command == "restart":
        await database.record_health_event(slug, "restart")
        metrics.record_container_lifecycle("restart", slug)
    elif body.command == "start":
        await database.record_health_event(slug, "start")
        metrics.record_container_lifecycle("start", slug)
    elif body.command == "remove":
        await database.remove_deployment(slug, worker_id=worker_id)
        await database.record_health_event(slug, "remove")
        metrics.record_container_lifecycle("remove", slug)
    return result


@app.get("/api/fleet/summary")
async def api_fleet_summary(request: Request) -> dict[str, Any]:
    """Aggregate fleet stats across local + all workers."""
    _require_reader(request)

    workers = await database.list_workers()
    total_services = 0
    total_running = 0
    online_workers = 0

    # Containers on workers that are NOT online. They are still running and
    # still earning -- an unreachable worker is a reporting gap, not a stopped
    # one -- so the fleet page can say how much of the estate these cards leave
    # out instead of letting a reboot read as containers lost (CashPilot-wij).
    unreachable_containers = 0
    unreachable_workers = 0

    for w in workers:
        _parse_worker_json(w)
        if w["status"] != "online":
            if w["container_count"]:
                unreachable_workers += 1
                unreachable_containers += w["container_count"]
            continue
        online_workers += 1
        total_services += w["container_count"]
        total_running += w["running_count"]

    return {
        "total_workers": len(workers),
        "online_workers": online_workers,
        "total_containers": total_services,
        "running_containers": total_running,
        "unreachable_containers": unreachable_containers,
        "unreachable_workers": unreachable_workers,
    }


@app.get("/api/fleet/api-key")
async def api_fleet_api_key(request: Request) -> dict[str, Any]:
    """Report whether a fleet API key is configured (owner only).

    Never returns the key value — use POST /api/fleet/api-key/reveal for that.
    """
    _require_owner(request)
    source = "env" if os.getenv("CASHPILOT_API_KEY") else ("file" if FLEET_API_KEY else "none")
    return {"is_set": bool(FLEET_API_KEY), "source": source}


@app.post("/api/fleet/api-key/reveal")
async def api_fleet_api_key_reveal(request: Request) -> dict[str, str]:
    """Reveal the configured fleet API key (owner only, audit-logged)."""
    user = _require_owner(request)
    logger.warning("Fleet key revealed by uid=%s", user.get("uid"))
    return {"api_key": FLEET_API_KEY or ""}


# ---------------------------------------------------------------------------
# Router groups (auth / pages / users)
#
# Imported and included LAST, after ``app`` and every shared symbol above is
# defined, so the handlers (which reference state via ``app.main.*``) resolve
# correctly. Kept here to preserve the public ``app.main`` test surface while
# splitting the low-regression route groups into app.routers.
# ---------------------------------------------------------------------------
from app.routers import activity as activity_router  # noqa: E402
from app.routers import auth as auth_router  # noqa: E402
from app.routers import importers as importers_router  # noqa: E402
from app.routers import pages as pages_router  # noqa: E402
from app.routers import users as users_router  # noqa: E402

app.include_router(activity_router.router)
app.include_router(auth_router.router)
app.include_router(pages_router.router)
app.include_router(users_router.router)
app.include_router(importers_router.router)


# The two import routes that need api_set_config and api_deploy. Their logic
# lives in app/routers/importers.py; they are declared here because no module
# may import main.
@app.post("/api/import/apply")
async def api_import_apply(request: Request, user: dict[str, Any] = Depends(_require_owner)) -> dict[str, Any]:
    async def set_collector_config(data: dict[str, str]) -> None:
        await api_set_config(request, ConfigUpdate(data=data), _auth=user)

    return await importers_router.apply(request, set_collector_config)


@app.post("/api/import/drafts/{slug}/deploy")
async def api_import_draft_deploy(
    request: Request, slug: str, worker_id: int | None = None, user: dict[str, Any] = Depends(_require_owner)
) -> dict[str, Any]:
    """Deploy an imported service through the normal deploy route, which does every check."""
    env = await importers_router.draft_env(slug)
    return await api_deploy(request, slug, DeployRequest(env=env), worker_id=worker_id, _auth=user)