from __future__ import annotations

import base64
import collections
import contextlib
import hashlib
import html
import io
import json
import logging
import math
import mimetypes
import os
import re
import secrets
import shutil
import sqlite3
import sys
import threading
import time
from contextlib import asynccontextmanager, contextmanager
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Callable, Generator, Iterable, Literal, Sequence, cast, overload
from urllib.parse import parse_qs, parse_qsl, quote, quote_plus, unquote, urlencode, urljoin, urlparse, urlsplit, urlunparse, urlunsplit

import feedparser
import httpx


def _parse_month_first_pubdate(date_string: str):
    # Handles non-RFC-2822 dates like "May 11, 2026 19:15:50 +0000" (no day-of-week).
    try:
        dt = datetime.strptime(date_string.strip(), "%B %d, %Y %H:%M:%S %z")
        return dt.utctimetuple()
    except ValueError:
        return None


feedparser.registerDateHandler(_parse_month_first_pubdate)

# The import block below was never isort-ordered; this suppresses I001 for the whole block.
from fastapi import FastAPI, Form, Query, Request  # noqa: I001
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, Response, StreamingResponse
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from PIL import Image as _PILImage
from readability import Document
from starlette.concurrency import run_in_threadpool
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.middleware.gzip import GZipMiddleware
from starlette.middleware.sessions import SessionMiddleware

from services import entry_content, escalation_reprobe
from services import bluesky
from services import flaresolverr as flaresolverr_service
from services import page_fetch
from services import full_content_fetch, itad_waitlist
from services import page_topics
from services.errors import public_error
from services.redirects import local_url
from services import site_content_plugins
from services import publish_date as publish_date_service
from services import deviantart as deviantart_service
from services import devto as devto_service
from services import youtube_oauth as youtube_oauth_service
from services import pinterest_oauth as pinterest_oauth_service
from services import quire as quire_service
from services import reddit as reddit_service
from services import inoreader as inoreader_service
from services import passwords
from services import podcast_audio
from services import podcast_feed_discovery
from services import content_edits
from services import link_canonical
from services import saved_articles as saved_articles_service
from services import saved_autofile as saved_autofile_service  # noqa: F401 -- re-exported: routes/saved.py does `from main import saved_autofile_service`
from services import archive_old_stars as archive_old_stars_service  # noqa: F401 -- re-exported, same as above
from services import refetch_batch
from services import unstar_tagged as unstar_tagged_service
from services import scraper_service
from services import html_sanitize
from services import tenancy
from services import youtube_embeds
from services import url_guard
from services.webhooks import WEBHOOK_VALID_FORMATS
from services.users import UserStore
from services.email import send_article_email, send_digest_email
from services import feed_discovery
from services.feed_discovery import discover_feed_urls_ex  # noqa: F401 -- re-exported: routes/feeds.py does `from main import discover_feed_urls_ex`
from services.feed_refresh import FeedRefreshService
from services.lead_images import LeadImageService, upgrade_image_size_param
from services.reader_api import ReaderApi
from services.starred_archive import StarredArchiveService
from services.youtube import YouTubeDurationService
from services.websub import WebSubService
from services.fever import FeverService
from services.greader import GReaderService
from services.miniflux import MinifluxService
from services.youtube_sync import sync_youtube_folder

from state import (
    _AUTOFETCH_HOST_COOLDOWN_S,
    _BROWSER_UA_CACHE_TTL,
    _DA_SYNC_MAX_AUTO_RESUMES,
    _FLARESOLVERR_FEEDS_CACHE_TTL,
    _INSTANCE_SETTING_TTL_SECONDS,
    _LAST_SEEN_THROTTLE_SECONDS,
    _MEDIA_SCAN_TTL_ERROR,
    _PROXY_DOWN_COOLDOWN_SECONDS,
    _PROXY_FEEDS_CACHE_TTL,
    _TAILSCALE_DOWN_COOLDOWN_SECONDS,
    _TAILSCALE_FEEDS_CACHE_TTL,
    _THUMB_FETCH_FAIL_CACHE,  # noqa: F401 -- re-exported: routes/system.py's /thumb proxy does `main._THUMB_FETCH_FAIL_CACHE`
    _THUMB_FETCH_FAIL_LOCK,  # noqa: F401 -- re-exported, same as above
    _THUMB_FETCH_FAIL_TTL,  # noqa: F401 -- re-exported, same as above
    _app_settings_cache,
    _app_settings_cache_lock,
    _autofetch_failed_hosts,
    _autofetch_hosts_lock,
    _browser_ua_cache,
    _browser_ua_cache_at,
    _browser_ua_cache_lock,
    _da_sync_active,
    _da_sync_lock,
    _enhancement_inflight_feeds,
    _enhancement_inflight_lock,
    _feed_last_post_cache,
    _feed_last_post_cache_lock,
    _flaresolverr_feeds_cache,
    _flaresolverr_feeds_cache_at,
    _flaresolverr_feeds_cache_lock,
    _home_request_semaphore,  # noqa: F401 -- re-exported: routes/home.py's `home` route does `main._home_request_semaphore`
    _instance_setting_cache,
    _instance_setting_cache_lock,
    _last_seen_touch,
    _last_seen_touch_lock,
    _login_failures,  # noqa: F401 -- re-exported: routes/system.py's login rate-limit helpers do `main._login_failures`
    _login_failures_lock,  # noqa: F401 -- re-exported, same as above
    _media_scan_in_progress,
    _media_scan_lock,
    _prefetch_header_log_lock,
    _prefetch_header_log_remaining,
    _proxy_down_lock,
    _proxy_down_until,
    _proxy_feeds_cache,
    _proxy_feeds_cache_at,
    _proxy_feeds_cache_lock,
    _scheduler_state,
    _scheduler_state_lock,
    _tag_alias_cache,
    _tag_alias_lock,
    _tailscale_down_lock,
    _tailscale_down_until,
    _tailscale_feeds_cache,
    _tailscale_feeds_cache_at,
    _tailscale_feeds_cache_lock,
    FEED_LAST_POST_TTL_SECONDS,
    updating_feeds,
    updating_feeds_lock,
    _PerUserDict,  # noqa: F401 -- re-exported: routes/integrations_youtube.py and tests still do `main._PerUserDict`
    _ad_asset_hashes_cache,
    _ad_asset_hashes_lock,
    _autofetch_jobs,
    _autofetch_jobs_lock,
    _AUTOFETCH_JOB_STALE_S,
    _has_manual_tags_cache,
    _has_manual_tags_lock,
    HAS_MANUAL_TAGS_CACHE_TTL_SECONDS,
    _meta_structure_cache,
    _meta_structure_lock,
    _problematic_feeds_cache,
    _problematic_feeds_cache_lock,
    PROBLEMATIC_FEEDS_CACHE_TTL_SECONDS,
    PROBLEMATIC_FEEDS_LIMIT,
    _refetch_jobs,
    _refetch_jobs_lock,
    feed_title_map_cache,
    feed_title_map_cache_lock,
    FEED_TITLE_MAP_CACHE_TTL_SECONDS,
    tag_counts_cache,
    tag_counts_cache_lock,
    TAG_COUNTS_CACHE_TTL_SECONDS,
    unread_counts_cache,
    unread_counts_cache_lock,
    UNREAD_COUNTS_CACHE_TTL_SECONDS,
    _bump_unread_counts_generation,
    get_unread_counts_generation,
    invalidate_unread_counts_cache,
    try_start_unread_refresh,
    clear_unread_refresh_inflight,
    next_refresh_rotation_offset,
    check_manual_refresh_cooldown,
    unread_counts_compute_lock,
)

BASE_DIR = Path(__file__).resolve().parent
# Dedicated app logger. Previously this piggybacked on "uvicorn.error", but
# uvicorn owns that logger and (depending on startup/rollover timing) can leave
# it not propagating to our root file handler — silently dropping every app log
# line ([maintenance], [refresh], …) even though the code ran. "lectio" is our
# own namespace that nothing else reconfigures, so it always reaches root.
LOGGER = logging.getLogger("lectio")
LOGGER.setLevel(logging.INFO)

# Refresh and page-fetch threads keep this one process's GIL busy, and a request makes many short DB calls that each
# have to win it back (5ms per turn by default) -- measured 2026-10-02 as the same steps taking ~50ms at rest and 1-4s
# during a refresh burst. A shorter interval hands it back sooner; throughput cost is small.
try:
    sys.setswitchinterval(float(os.getenv("LECTIO_GIL_SWITCH_INTERVAL", "0.001")))
except ValueError:
    pass


class _ReaderNonFatalParseWarningFilter(logging.Filter):
    """Filter known non-fatal feed parsing warnings from `reader` logs."""

    def filter(self, record: logging.LogRecord) -> bool:
        message = record.getMessage()
        if not message.startswith("parse "):
            return True

        suppressed_markers = (
            "got NonXMLContentType(",
            "got CharacterEncodingOverride(",
        )
        return not any(marker in message for marker in suppressed_markers)


class _ReaderUpdateFetchErrorFilter(logging.Filter):
    """Demote reader's per-feed fetch/parse failures from ERROR+traceback to a
    one-line WARNING.

    reader logs every failed ``update_feed`` at ERROR with a full stack trace via
    the ``reader.update`` logger — but Lectio already catches each one, applies
    backoff, records ``feed_failure_state.last_error``, and surfaces it in the
    problematic-feeds panel. So a routine 429/403/timeout/dead-feed produced a
    scary red traceback in the log (and the new Admin → Logs tab) for a condition
    that's fully handled. Keep the one-line summary as a WARNING but drop the
    redundant traceback; genuinely unexpected reader errors stay loud (ERROR)."""

    # Signatures of expected, Lectio-handled fetch/parse failures.
    _EXPECTED = (
        "bad http status code",  # any HTTP error status (429/403/404/410/5xx)
        "too many requests",
        "forbidden",
        "not found",
        "gone",
        "service unavailable",
        "timed out",
        "timeout",
        "connecttimeout",
        "readtimeout",
        "connectionerror",
        "connection reset",
        "connection aborted",
        "name or service not known",
        "temporary failure in name resolution",
        "certificate",
        "ssl",
        "tlsv1",
        "nonxmlcontenttype",
        "no parser",
        "document declared as",
        "not well-formed",
    )

    def filter(self, record: logging.LogRecord) -> bool:
        if record.levelno < logging.ERROR:
            return True
        message = record.getMessage().lower()
        if "status=error" not in message and "updating feed" not in message:
            return True
        if any(marker in message for marker in self._EXPECTED):
            record.levelno = logging.WARNING
            record.levelname = "WARNING"
            record.exc_info = None
            record.exc_text = None
        return True


def _configure_reader_logging() -> None:
    if os.getenv("LECTIO_SUPPRESS_READER_PARSE_WARNINGS", "1") != "1":
        return
    logging.getLogger("reader").addFilter(_ReaderNonFatalParseWarningFilter())
    logging.getLogger("reader.update").addFilter(_ReaderUpdateFetchErrorFilter())


# Query-string secrets that reach the log in request URLs: httpx logs every outgoing URL (the YouTube Data API takes its key as
# ``&key=``, signed image URLs carry a signature) and the access log records incoming ones (OAuth callbacks carry ``code``/``state``).
_LOG_SECRET_RE = re.compile(
    r"([?&](?:key|api_key|access_token|refresh_token|client_secret|token|code|state|password|jwt|sig|signature"
    r"|x-amz-signature|x-amz-security-token)=)[^&\s\"']+",
    re.IGNORECASE,
)


class _RedactingFormatter(logging.Formatter):
    """Log formatter that masks URL query-string secrets before the line is written anywhere."""

    def format(self, record: logging.LogRecord) -> str:
        return _LOG_SECRET_RE.sub(r"\1<redacted>", super().format(record))


def _configure_persistent_logging() -> None:
    """Attach a rotating file handler when LECTIO_LOG_DIR is set.

    Defaults: 5 MB per file, 5 backups (LECTIO_LOG_MAX_BYTES, LECTIO_LOG_BACKUPS).
    This is the durable record; :func:`_configure_console_logging` mirrors the same
    stream to the console. Without LECTIO_LOG_DIR only the console handler runs.
    """
    log_dir_str = os.getenv("LECTIO_LOG_DIR", "").strip()
    if not log_dir_str:
        return
    from logging.handlers import RotatingFileHandler

    log_dir = Path(log_dir_str)
    log_dir.mkdir(parents=True, exist_ok=True)
    max_bytes = int(os.getenv("LECTIO_LOG_MAX_BYTES", str(5 * 1024 * 1024)))
    backup_count = int(os.getenv("LECTIO_LOG_BACKUPS", "5"))
    handler = RotatingFileHandler(
        log_dir / "lectio.log",
        maxBytes=max_bytes,
        backupCount=backup_count,
        encoding="utf-8",
    )
    handler.setFormatter(_RedactingFormatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
    # Attach to the root logger so it captures everything (uvicorn, reader, app).
    root = logging.getLogger()
    if root.level > logging.INFO or root.level == logging.NOTSET:
        root.setLevel(logging.INFO)
    root.addHandler(handler)


def _configure_console_logging() -> None:
    """Mirror everything the app logs to the console, so `docker compose logs` shows it.

    uvicorn installs handlers on its own loggers with propagate=False, so its startup lines appear on the
    console while every app, reader and library log record propagates to the root logger — which, in the
    container, owns only the rotating file handler under LECTIO_LOG_DIR. The result was that all app logging
    was invisible to `docker compose logs` and readable only from inside the container. That cost hours during
    the 2026-08-18 restart hunt: the scheduler watchdog logged two ERROR lines before every exit and the
    container log showed nothing but a silent restart.

    Writes to stderr, which is where uvicorn's own handlers write, so the interleaving stays sane. The check
    below excludes FileHandler because RotatingFileHandler is itself a StreamHandler subclass.
    """
    root = logging.getLogger()
    for existing in root.handlers:
        if isinstance(existing, logging.StreamHandler) and not isinstance(existing, logging.FileHandler):
            if not isinstance(existing.formatter, _RedactingFormatter):
                existing.setFormatter(_RedactingFormatter(existing.formatter._fmt if existing.formatter else None))
            return
    handler = logging.StreamHandler()
    handler.setFormatter(_RedactingFormatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
    # INFO by default, because the whole point is that app logs become visible. But loosening the root
    # level is a decision about someone else's process, so LECTIO_LOG_LEVEL takes precedence when set —
    # a deployment that wants a quieter console says so instead of being overruled.
    wanted = os.getenv("LECTIO_LOG_LEVEL", "").strip().upper()
    if wanted:
        root.setLevel(getattr(logging, wanted, logging.INFO))
    elif root.level > logging.INFO or root.level == logging.NOTSET:
        root.setLevel(logging.INFO)
    root.addHandler(handler)


def _configure_access_log_filter() -> None:
    # Kept for backward-compat with previous wiring; the actual access log is
    # now emitted by _AccessLogMiddleware (uvicorn's --no-access-log disables
    # the built-in one whose filters we couldn't reliably hook).
    pass


def _attach_pending_access_filter() -> None:
    pass


_configure_reader_logging()
_configure_console_logging()
_configure_persistent_logging()
_configure_access_log_filter()


def _load_local_env(env_path: str | Path | None = None) -> None:
    """Load simple KEY=VALUE pairs from a .env file into os.environ if not already set.

    This is intentionally minimal (no interpolation) and avoids adding a dependency.
    """
    candidates = []
    if env_path:
        candidates.append(Path(env_path))
    candidates.append(BASE_DIR / ".env")
    candidates.append(Path.cwd() / ".env")
    for p in candidates:
        try:
            if not p or not p.exists():
                continue
            with p.open("r", encoding="utf-8") as fh:
                for raw in fh:
                    line = raw.strip()
                    if not line or line.startswith("#"):
                        continue
                    if "=" not in line:
                        continue
                    key, val = line.split("=", 1)
                    key = key.strip()
                    val = val.strip()
                    if (val.startswith('"') and val.endswith('"')) or (val.startswith("'") and val.endswith("'")):
                        val = val[1:-1]
                    # Do not overwrite existing environment vars
                    if key and os.getenv(key) is None:
                        os.environ[key] = val
            break
        except Exception:
            continue


# Load local .env early so os.getenv() below can pick up values provided by the developer
_load_local_env()
# Default to BASE_DIR/data (matching the Docker ./data:/data mapping) so a local
# run without LECTIO_DATA_DIR set writes DBs and service dirs (scraped-feeds/,
# deviantart-feeds/) into ./data instead of polluting the repo root.
# expanduser so a tilde path (LECTIO_DATA_DIR=~/.lectio) resolves against $HOME
# rather than being treated as a literal "./~" segment.
_data_dir_env = os.getenv("LECTIO_DATA_DIR")
DATA_DIR = Path(os.path.expanduser(_data_dir_env) if _data_dir_env else BASE_DIR / "data").resolve()
DATA_DIR.mkdir(parents=True, exist_ok=True)
THUMB_DB_PATH = DATA_DIR / "lectio_thumb_cache.sqlite"
# Global, content-addressed (source-URL hash -> original image bytes) cache for the
# /api/img proxy. Like the thumb cache, it holds no per-user data and is NOT routed
# through tenancy. See ensure_img_cache_schema / api_img_proxy.
IMG_CACHE_DB_PATH = DATA_DIR / "lectio_img_cache.sqlite"
# Global, content-addressed (video_id -> duration): a fact about the video, not
# per-user, so it's shared — a user without a YouTube key still gets durations
# another user's key already fetched. Like the thumb cache, NOT routed through tenancy.
YT_DURATION_DB_PATH = DATA_DIR / "lectio_yt_durations.sqlite"
# Global account registry (NOT per-user, NOT routed through tenancy): one users
# table for the whole instance.
AUTH_DB_PATH = DATA_DIR / "lectio_auth.sqlite"
THUMB_CACHE_DIR = DATA_DIR / "thumb_cache"  # legacy on-disk cache; entries migrate lazily on access

# Bind the tenancy resolver: per-user DBs live under DATA_DIR/users/{uid}/. The thumb
# cache (THUMB_DB_PATH) is intentionally NOT routed through tenancy — it is a
# content-addressed global cache shared across all users. See services/tenancy.py.
tenancy.configure(data_dir=DATA_DIR)

ROOT_FOLDER_NAME = "All Feeds"
_LECTIO_FOLDER_NAME = "_Lectio"

# Virtual, derived folder pinned to the bottom of the sidebar that holds every
# subscribed feed with no folder_feeds row (e.g. feeds imported via OPML/reader
# migration that never got a folder). It has no DB row; its id is a negative
# sentinel so it never collides with real (positive) folder ids, and its
# membership is computed per-render as (all reader feeds − foldered feeds).
UNCATEGORIZED_FOLDER_ID = -1
UNCATEGORIZED_FOLDER_NAME = "Uncategorized"

scraper_service.init(DATA_DIR)
deviantart_service.init(DATA_DIR)
devto_service.init(DATA_DIR)


def _da_seed_lead_image(feed_url: str, entry_id: str, image_url: str) -> None:
    """Sink for the DeviantArt service: store the API image as the entry's lead
    image (DB + live cache) unless one is already cached (preserve customizations)."""
    try:
        if not lead_image_service.get_cached_entry_thumbnail(feed_url, entry_id, ""):
            lead_image_service.store_entry_lead_image(feed_url, entry_id, image_url)
    except Exception:
        LOGGER.exception("[deviantart] failed to seed lead image for %s", entry_id)


deviantart_service.set_lead_image_sink(_da_seed_lead_image)


def _devto_seed_lead_image(feed_url: str, entry_id: str, image_url: str) -> None:
    """Sink for the dev.to service: store the API cover image as the entry's lead
    image (DB + live cache) unless one is already cached (preserve customizations)."""
    try:
        if not lead_image_service.get_cached_entry_thumbnail(feed_url, entry_id, ""):
            lead_image_service.store_entry_lead_image(feed_url, entry_id, image_url)
    except Exception:
        LOGGER.exception("[devto] failed to seed lead image for %s", entry_id)


devto_service.set_lead_image_sink(_devto_seed_lead_image)
DEFAULT_AUTO_REFRESH_MINUTES = 60
MIN_AUTO_REFRESH_MINUTES = 5
MANUAL_REFRESH_COOLDOWN_SECONDS = 60
FAILED_FEED_BACKOFF_BASE_SECONDS = 60
FAILED_FEED_BACKOFF_MAX_SECONDS = 60 * 60 * 24
AUTO_REFRESH_SETTING_KEY = "auto_refresh_minutes"
SORT_BY_SETTING_KEY = "sort_by"
SORT_DIR_SETTING_KEY = "sort_dir"
# Feeds and Saved remember their sort SEPARATELY. One shared pair meant picking
# an order in Saved silently changed Feeds, and vice versa — the two are
# different jobs (a publish-date backlog vs a to-do pile), so they want
# different orders. The unprefixed keys stay Feeds' so existing installs keep
# their remembered value.
SAVED_SORT_BY_SETTING_KEY = "saved_sort_by"
SAVED_SORT_DIR_SETTING_KEY = "saved_sort_dir"


def sort_setting_keys(star_only: bool) -> tuple[str, str]:
    """The (sort_by, sort_dir) setting keys for the active scope."""
    if star_only:
        return SAVED_SORT_BY_SETTING_KEY, SAVED_SORT_DIR_SETTING_KEY
    return SORT_BY_SETTING_KEY, SORT_DIR_SETTING_KEY


GLOBAL_NOTE_SETTING_KEY = "global_note"
PROBLEMATIC_FEEDS_LAST_VIEWED_AT_SETTING_KEY = "problematic_feeds_last_viewed_at"
YOUTUBE_SYNC_LAST_AT_KEY = "youtube_sync_last_at"
YOUTUBE_SYNC_LAST_RESULT_KEY = "youtube_sync_last_result"
EMAIL_TO_SETTING_KEY = "email_to"
EMAIL_BCC_SETTING_KEY = "email_bcc_address"
PROFILE_NAME_SETTING_KEY = "profile_name"
PROFILE_EMAIL_SETTING_KEY = "profile_email"
SETTING_TZ_DISPLAY = "tz_display"
SETTING_PORTRAIT_IMG_MAX_WIDTH = "portrait_img_max_width"
# Per-user: Ctrl/Cmd-click or middle-click on a post title (opens the article in a new tab without opening it in the reader) marks the
# post read. Off ("0") by default: those clicks leave it unread.
SETTING_MARK_READ_ON_NEW_TAB = "mark_read_on_new_tab"
# Per-user: rewrite every third-party <img src> in the article pane body to
# /api/img?u=… instead of loading it directly. Off ("0") by default — raises
# /api/img traffic and cache size, and drops srcset (so high-DPI screens fetch
# the proxied single-resolution src). Read Mode always does this regardless of
# this setting (see proxy_all_body_images).
SETTING_PROXY_BODY_IMAGES = "proxy_body_images"
# Outbound-fetch SOCKS5/HTTP proxy (e.g. a gluetun VPN container). URL is
# admin-only (a non-admin choosing an arbitrary proxy target is SSRF-adjacent);
# mode is per-user-overridable — "" means inherit the instance default. See
# get_proxy_url / get_proxy_mode.
SETTING_PROXY_URL = "proxy_url"
SETTING_PROXY_MODE = "proxy_mode"  # "off" / "as_needed" / "always"
# Last-resort outbound proxy (e.g. a Tailscale exit node reaching the real home
# IP) — a different trust tier from SETTING_PROXY_URL: real exposure cost, not a
# disposable datacenter NAT. Rides the SAME proxy_mode as the primary proxy
# rather than its own mode (Josh's own framing: mirror the existing pattern,
# not invent a new one) — its actual gate is simply being unconfigured by
# default, same as the primary proxy. Only ever reached via as_needed's
# per-feed escalation, one rung past the primary proxy, never via "always"
# (that would route every fetch through the home IP, which "last resort" rules
# out). URL is admin-only for the same SSRF-adjacent reason as SETTING_PROXY_URL.
SETTING_TAILSCALE_URL = "tailscale_url"
# FlareSolverr (real headless Chrome, purpose-built to solve Cloudflare/DDoS-
# Guard style challenges) — one rung past the primary proxy, before the
# last-resort backend above. Rides the same proxy_mode too. No token/auth (an
# internal-network-only container); zero personal exposure — always stacked
# with the primary proxy (SETTING_PROXY_URL) when one is configured, never
# with the last-resort one, since it doesn't need the home IP to work and
# spending it here would defeat "last resort". Gated narrower than the other
# two tiers: only a real challenge page (bot_challenge.FeedBlockedError, see
# FeedRefreshService._is_bot_challenge), not any refusal — a plain IP-block
# 403 wants a different exit IP, not a browser solving a challenge that was
# never served.
SETTING_FLARESOLVERR_URL = "flaresolverr_url"
SETTING_MAINTENANCE_HOUR = "maintenance_hour"
SETTING_IMG_CACHE_DAYS = "img_cache_days"
SETTING_IMG_CACHE_MAX_DIM = "img_cache_max_dim"
SETTING_IMG_TARGET_BYTES = "img_target_bytes"
SETTING_YT_API_KEY = "yt_api_key"
SETTING_YT_CHANNEL_ID = "yt_channel_id"
SETTING_YT_FOLDER_NAME = "yt_folder_name"
# Per-user: use the standard youtube.com embed host (account features — Share /
# Watch Later — work) instead of the privacy-enhanced youtube-nocookie.com. Off
# (privacy) by default; the standard host sets YouTube cookies. "1"/"0".
SETTING_YT_EMBED_ACCOUNT_FEATURES = "yt_embed_account_features"
# Global per-user toggle: auto-mark YouTube Shorts read across ALL YouTube feeds at
# refresh, regardless of the per-feed hide_shorts pref. Off ("0") by default.
SETTING_YT_HIDE_SHORTS_GLOBAL = "yt_hide_shorts_global"
# Global per-user toggle: don't show YouTube videos that haven't premiered yet
# (liveBroadcastContent == "upcoming") across ALL YouTube feeds, regardless of
# the per-feed hide_unpremiered pref. Off ("0") by default.
SETTING_YT_HIDE_UNPREMIERED_GLOBAL = "yt_hide_unpremiered_global"
# Global per-user toggle: auto-mark members-only videos read across ALL YouTube
# feeds at refresh, regardless of the per-feed hide_members_only pref. Off
# ("0") by default. Detection has a real per-video cost (a watch-page fetch —
# unlike hide_shorts/hide_unpremiered, membership status isn't in the Data API
# response already fetched for duration/premiere status), so it only runs for
# entries that have never been checked before (see youtube_video_duration's
# members_only column).
SETTING_YT_HIDE_MEMBERS_ONLY_GLOBAL = "yt_hide_members_only_global"
# Global per-user toggle: hide a webcomic strip a publisher has locked behind a
# supporter tier (cad-comic.com and the like) until its own stated unlock date,
# across ALL webcomic feeds, regardless of the per-feed hide_locked_comics pref.
# Not YouTube-specific — see LeadImageService.check_and_cache_webcomic_lock.
# Off ("0") by default.
SETTING_HIDE_LOCKED_COMICS_GLOBAL = "hide_locked_comics_global"
# Daily YouTube Data API quota cap (units). Google's default is 10,000/day; make it
# a setting in case a higher quota is granted.
SETTING_YT_QUOTA_CAP = "yt_quota_cap"
YT_QUOTA_DEFAULT_CAP = 10000
# Watermark for the auto-add-to-playlist rule's "what's new since I last
# checked" scan — see _run_youtube_playlist_rules_after_refresh.
SETTING_YT_PLAYLIST_AUTO_LAST_CHECK = "yt_playlist_auto_last_check"
# "On star, also send to…" — per-user destinations fired when an article is starred.
SETTING_STAR_SEND_INSTAPAPER = "star_send_instapaper"  # "1"/"0"
SETTING_STAR_SEND_YT_PLAYLIST = "star_send_yt_playlist"  # playlist id ("" = off)
SETTING_STAR_SEND_YT_PLAYLIST_TITLE = "star_send_yt_playlist_title"
SETTING_STAR_SEND_EMAIL = "star_send_email"  # address ("" = off)
SETTING_RESEND_API_KEY = "resend_api_key"
SETTING_EMAIL_FROM = "email_from"
SETTING_INSTAPAPER_USERNAME = "instapaper_username"
SETTING_INSTAPAPER_PASSWORD = "instapaper_password"
SETTING_DEVIANTART_CLIENT_ID = "deviantart_client_id"
SETTING_DEVIANTART_CLIENT_SECRET = "deviantart_client_secret"
SETTING_DEVIANTART_ACCESS_TOKEN = "deviantart_access_token"
SETTING_DEVIANTART_REFRESH_TOKEN = "deviantart_refresh_token"
SETTING_DEVIANTART_TOKEN_EXPIRES_AT = "deviantart_token_expires_at"
SETTING_DEVIANTART_USERNAME = "deviantart_username"
SETTING_DEVIANTART_OAUTH_STATE = "deviantart_oauth_state"
SETTING_DEVIANTART_OAUTH_VERIFIER = "deviantart_oauth_verifier"
SETTING_DEVIANTART_FOLDER_NAME = "deviantart_folder_name"
SETTING_DEVIANTART_SYNC_STATUS = "deviantart_sync_status"
# Structured companion to the status string: JSON {failed:[{username,error}], unwatched:[username]}
# so the Settings UI can list the failed/unwatched artists as profile links instead
# of telling the user to "see logs" (which they have no access to).
SETTING_DEVIANTART_SYNC_DETAIL = "deviantart_sync_detail"
# Set "1" the moment a sync finds a NEWLY-unwatched (now auto-paused) artist;
# cleared only when the DeviantArt Integrations sub-tab is actually opened —
# drives the Integrations/DeviantArt tab dots, deliberately separate from
# whether the unwatched list itself is empty (that list stays populated for
# Josh to act on "at some point or not"; this flag is just "there's something
# new in it you haven't looked at yet").
SETTING_DEVIANTART_UNWATCHED_DIRTY = "deviantart_unwatched_dirty"
SETTING_YT_OAUTH_CLIENT_ID = "yt_oauth_client_id"
SETTING_YT_OAUTH_CLIENT_SECRET = "yt_oauth_client_secret"
SETTING_YT_OAUTH_ACCESS_TOKEN = "yt_oauth_access_token"
SETTING_YT_OAUTH_REFRESH_TOKEN = "yt_oauth_refresh_token"
SETTING_YT_OAUTH_TOKEN_EXPIRES_AT = "yt_oauth_token_expires_at"
SETTING_YT_OAUTH_STATE = "yt_oauth_state"
# Shared-instance YouTube OAuth creds stored in the admin's app_settings.
SETTING_SHARED_YT_OAUTH_CLIENT_ID = "shared_yt_oauth_client_id"
SETTING_SHARED_YT_OAUTH_CLIENT_SECRET = "shared_yt_oauth_client_secret"
# Pinterest outbound (save / pin an article to a board).
SETTING_PINTEREST_OAUTH_CLIENT_ID = "pinterest_oauth_client_id"
SETTING_PINTEREST_OAUTH_CLIENT_SECRET = "pinterest_oauth_client_secret"
SETTING_PINTEREST_OAUTH_ACCESS_TOKEN = "pinterest_oauth_access_token"
SETTING_PINTEREST_OAUTH_REFRESH_TOKEN = "pinterest_oauth_refresh_token"
SETTING_PINTEREST_OAUTH_TOKEN_EXPIRES_AT = "pinterest_oauth_token_expires_at"
SETTING_PINTEREST_OAUTH_STATE = "pinterest_oauth_state"
# Shared-instance Pinterest OAuth creds stored in the admin's app_settings.
SETTING_SHARED_PINTEREST_OAUTH_CLIENT_ID = "shared_pinterest_oauth_client_id"
SETTING_SHARED_PINTEREST_OAUTH_CLIENT_SECRET = "shared_pinterest_oauth_client_secret"
# Inoreader migration — OAuth tokens and import checkpoint.
SETTING_INOREADER_CLIENT_ID = "inoreader_client_id"
SETTING_INOREADER_CLIENT_SECRET = "inoreader_client_secret"
SETTING_INOREADER_ACCESS_TOKEN = "inoreader_access_token"
SETTING_INOREADER_REFRESH_TOKEN = "inoreader_refresh_token"
SETTING_INOREADER_TOKEN_EXPIRES_AT = "inoreader_token_expires_at"
SETTING_INOREADER_OAUTH_STATE = "inoreader_oauth_state"
SETTING_INOREADER_IMPORT_STATE = "inoreader_import_state"
SETTING_INOREADER_EXPORT_DIR = "inoreader_export_dir"  # server-side path to JSON export files
# Miniflux migration (FROM a Miniflux instance — distinct from the Miniflux API compat server).
SETTING_MINIFLUX_IMPORT_URL = "miniflux_import_url"
SETTING_MINIFLUX_IMPORT_TOKEN = "miniflux_import_token"
SETTING_MINIFLUX_IMPORT_STATE = "miniflux_import_state"
# FreshRSS migration.
SETTING_FRESHRSS_URL = "freshrss_url"
SETTING_FRESHRSS_USERNAME = "freshrss_username"
SETTING_FRESHRSS_PASSWORD = "freshrss_password"
SETTING_FRESHRSS_IMPORT_STATE = "freshrss_import_state"
# tt-rss migration.
SETTING_TTRSS_URL = "ttrss_url"
SETTING_TTRSS_USERNAME = "ttrss_username"
SETTING_TTRSS_PASSWORD = "ttrss_password"
SETTING_TTRSS_IMPORT_STATE = "ttrss_import_state"
# Reddit OAuth integration.
SETTING_REDDIT_CLIENT_ID = "reddit_client_id"
SETTING_REDDIT_CLIENT_SECRET = "reddit_client_secret"
SETTING_REDDIT_ACCESS_TOKEN = "reddit_oauth_access_token"
SETTING_REDDIT_REFRESH_TOKEN = "reddit_oauth_refresh_token"
SETTING_REDDIT_TOKEN_EXPIRES_AT = "reddit_oauth_token_expires_at"
SETTING_REDDIT_OAUTH_STATE = "reddit_oauth_state"
SETTING_REDDIT_USERNAME = "reddit_username"
SETTING_SHARED_REDDIT_CLIENT_ID = "shared_reddit_client_id"
SETTING_SHARED_REDDIT_CLIENT_SECRET = "shared_reddit_client_secret"
SETTING_STAR_SEND_REDDIT_SUBREDDIT = "star_send_reddit_subreddit"
# Instance tuning settings (admin-only, stored in admin's app_settings).
SETTING_FETCH_HISTORY_MAX_AGE_DAYS = "fetch_history_max_age_days"
# Auto-read age (days; 0 = off) at instance and account level. Each level below can only shorten it: folder, then feed.
SETTING_AUTO_READ_DAYS_INSTANCE = "auto_read_days_instance"
SETTING_AUTO_READ_DAYS_USER = "auto_read_days_user"
SETTING_TOMBSTONE_SWEEP_DAYS = "tombstone_sweep_days"
SETTING_LOGIN_MAX_FAILURES = "login_max_failures"
SETTING_LOGIN_WINDOW_SECONDS = "login_window_seconds"
SETTING_DEFAULT_AUTO_REFRESH_MINUTES = "default_auto_refresh_minutes"
# Quire outbound (turn an article into a task in a chosen Quire project).
SETTING_QUIRE_CLIENT_ID = "quire_client_id"
SETTING_QUIRE_CLIENT_SECRET = "quire_client_secret"
SETTING_QUIRE_ACCESS_TOKEN = "quire_access_token"
SETTING_QUIRE_REFRESH_TOKEN = "quire_refresh_token"
SETTING_QUIRE_TOKEN_EXPIRES_AT = "quire_token_expires_at"
SETTING_QUIRE_OAUTH_STATE = "quire_oauth_state"
SETTING_QUIRE_USERNAME = "quire_username"
SETTING_QUIRE_PROJECT_OID = "quire_project_oid"  # default destination project ("" = none)
SETTING_QUIRE_PROJECT_NAME = "quire_project_name"  # cached display name of that project
SETTING_STAR_SEND_QUIRE = "star_send_quire"  # "1"/"0"
# Quire rate limits are per-organization, per-minute and per-hour (Free: 50/min, 200/hr).
# These are Lectio's own sliding-window caps used to drive the usage meter + back off.
SETTING_QUIRE_RATE_CAP_MIN = "quire_rate_cap_min"
SETTING_QUIRE_RATE_CAP_HOUR = "quire_rate_cap_hour"
SETTING_QUIRE_PLAN = "quire_plan"  # detected plan name of the destination project's org (display only)
QUIRE_RATE_DEFAULT_CAP_MIN = 50
QUIRE_RATE_DEFAULT_CAP_HOUR = 200
AUTO_REFRESH_OPTION_MINUTES = (0, 5, 15, 30, 60, 360, 720)
SCHEDULER_POLL_SECONDS = 30
DEFAULT_SORT_BY = "post"
DEFAULT_SORT_DIR = "asc"
CHUNK_SIZE = 20
# feed_fetch_history retention configured via the Administration panel.
READABILITY_USER_AGENT = "Lectio/0.1 (+https://localhost)"
# Honest identifier for outbound fetches — names the app and links to the repo,
# the good-citizen behavior some hosts (e.g. rachelbythebay.com) explicitly
# reward. This is the DEFAULT for podcast discovery/borrow fetches.
LECTIO_HONEST_USER_AGENT = "Lectio/0.1 (+https://github.com/joshg253/Lectio)"
# Some podcast-host feeds (Buzzsprout, Libsyn, …) sit behind Cloudflare, which
# 403s any non-browser UA. We escalate to this browser UA ONLY after the honest
# request is actually refused (see _polite_safe_get) — never preemptively, so we
# don't spoof hosts that are happy to serve Lectio.
PODCAST_FETCH_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
# In-memory cache of domains known to have Cross-Origin-Resource-Policy restrictions.
# Values: True = same-site/same-origin (proxy needed), False = no restriction.
_CORP_DOMAIN_CACHE: dict[str, bool] = {}

# Hosts that serve a "this image was hotlinked" placeholder when the request
# carries a foreign Referer (or no same-origin signal). referrerpolicy="no-referrer"
# defeats this for *fresh* image loads, but a browser that cached the placeholder
# under the (unchanged) image URL keeps serving it. Routing these hosts' images
# through the /api/img proxy gives the browser a new same-origin URL it hasn't
# cached, and the server fetch carries no Referer — so the real image loads and
# stays correct. Add a registrable domain (matches it and any subdomain).
# wixmp.com serves DeviantArt images behind short-lived signed (?token=) URLs;
# proxying caches the bytes server-side so the article image survives token expiry.
# private-user-images.githubusercontent.com serves GitHub release/issue screenshots
# behind short-lived (~5 min) JWT-signed URLs — same problem, same fix.
# fabiensanglard.net is the *inverse* hotlink case: it 403s an image fetched with
# no Referer (e.g. its .webp files) but serves it with a same-origin Referer. The
# browser can't send fabiensanglard's own origin as Referer, so a direct <img>
# load fails (its .webp break in reader/web view while its .jpg loads); routing
# through /api/img lets the proxy's same-origin-Referer retry (api_img_proxy)
# fetch the real bytes. See ARCHITECTURE "Same-origin Referer".
_HOTLINK_IMG_HOSTS: frozenset[str] = frozenset(
    {"nanolx.org", "wixmp.com", "private-user-images.githubusercontent.com", "fabiensanglard.net"}
)


def _is_hotlink_img_host(netloc: str) -> bool:
    host = (netloc or "").split("@")[-1].split(":")[0].lower()
    return any(host == h or host.endswith("." + h) for h in _HOTLINK_IMG_HOSTS)


MANUAL_TAG_KEY_PREFIX = "lectio.manual_tag."
MAX_MANUAL_TAGS = 12
# Hard cap on how many of an entry's feed tags are carried to the page. RPS
# ships 28 on a single post, so this is generous on purpose — the cost of one
# extra chip is that it is ignored, and the cost of a missing one is that the
# tag cannot be filtered on at all.
MAX_FEED_TAG_SUGGESTIONS = 40
# How many chips the row shows before the "+N more" expander. Eight fits the
# header without wrapping; the rest are one click away rather than absent.
FEED_TAG_CHIPS_COLLAPSED = 8
TAG_VALUE_PATTERN = re.compile(r"^[A-Za-z0-9_.#+][A-Za-z0-9_.#+-]{0,31}$")


def _static_asset_version() -> str:
    """Cache-buster for every ?v=-versioned static asset.

    Hashes ALL css/js under static/ rather than a hand-kept list. The list went
    stale exactly as you would expect: offline-probe.js and js/cleanup.js are
    both served with ?v={{ static_asset_version }} and neither was in it, so
    editing them left the version unchanged and browsers — and the service
    worker, which caches /static by design — kept serving the previous file.
    Deploying a fix and watching the old copy run is a bad way to spend an hour.

    `.webmanifest` is in the set for the same reason: it is served with the same
    ?v= and the same immutable, year-long Cache-Control, so a manifest edit that
    did not move the digest would be pinned in the browser indefinitely.

    Sorted so the digest is stable across filesystems.
    """
    try:
        _static = Path(__file__).parent / "static"
        paths = sorted(p for p in _static.rglob("*") if p.is_file() and p.suffix in (".css", ".js", ".webmanifest"))
        combined = b"".join(p.read_bytes() for p in paths)
        return hashlib.md5(combined).hexdigest()[:10]
    except Exception:
        return "dev"


STATIC_ASSET_VERSION = os.getenv("LECTIO_ASSET_VERSION") or _static_asset_version()
REFRESH_DEBUG_ENABLED = os.getenv("LECTIO_REFRESH_DEBUG", "0") == "1"
# Gates the slow-SQL timing/progress-step instrumentation (Plan.md Tier 1
# refresh-contention item) on both the meta and reader DB connections. Off by
# default: a sqlite3 progress handler fires every 1000 VM instructions on
# every statement, and the timing wrapper adds a perf_counter() pair per call
# -- real, if small, overhead on every single query, all the time, not worth
# paying outside an active investigation. Flip on when chasing a live stall.
LECTIO_PERF_DEBUG = os.getenv("LECTIO_PERF_DEBUG", "0") == "1"


def _env_int(name: str, default: int) -> int:
    try:
        return int(os.getenv(name, str(default)))
    except ValueError:
        return default


# /api/img proxy cache tuning (instance-level; admins can override in the
# Administration page, which takes precedence over these env fallbacks).
#   LECTIO_IMG_CACHE_DAYS    last-accessed TTL in days; 0 = keep forever (default 90)
#   LECTIO_IMG_CACHE_MAX_DIM longest-side px to downscale stored images to (default 3840)
_ENV_IMG_CACHE_DAYS = _env_int("LECTIO_IMG_CACHE_DAYS", 90)
_ENV_IMG_CACHE_MAX_DIM = _env_int("LECTIO_IMG_CACHE_MAX_DIM", 3840)
# Byte budget for a cached image. The dimension cap above is not a size cap — a
# 4K RGBA PNG sits exactly at MAX_DIM and still weighs ~12 MB, which is then
# served whole on every view. Over this, the image is re-encoded to WebP
# (lossless first, so screenshots and line art keep their edges). 0 disables.
_ENV_IMG_TARGET_BYTES = _env_int("LECTIO_IMG_TARGET_BYTES", 1_500_000)
# Max width (px) for portrait (taller-than-wide) article images; 0 = disabled.
_ENV_PORTRAIT_IMG_MAX_WIDTH = _env_int("LECTIO_PORTRAIT_IMG_MAX_WIDTH", 650)
# Outbound-fetch proxy endpoint (e.g. socks5h://gluetun:1080). "" = none configured.
_ENV_PROXY_URL = os.getenv("LECTIO_PROXY_URL", "").strip()
# Last-resort outbound proxy (e.g. socks5h://tailscale:1080). "" = none configured.
_ENV_TAILSCALE_URL = os.getenv("LECTIO_TAILSCALE_URL", "").strip()
# FlareSolverr endpoint (e.g. http://flaresolverr:8191/v1). "" = none configured.
_ENV_FLARESOLVERR_URL = os.getenv("LECTIO_FLARESOLVERR_URL", "").strip()

# --- Scheduler resilience (see ARCHITECTURE "Refresh scheduler") ---
# Feed fetches are strictly sequential, so one host that accepts a connection and
# then goes quiet delays every feed behind it. These bound that.
#   LECTIO_FEED_CONNECT_TIMEOUT  seconds to establish a feed connection
#   LECTIO_FEED_READ_TIMEOUT     seconds to wait for any single read from a feed host
FEED_CONNECT_TIMEOUT_SECONDS = float(_env_int("LECTIO_FEED_CONNECT_TIMEOUT", 10))
FEED_READ_TIMEOUT_SECONDS = float(_env_int("LECTIO_FEED_READ_TIMEOUT", 30))
# A read deadline is per-socket-read, not total: a host trickling one byte at a
# time never trips it. The watchdog is the backstop — it watches for PROGRESS,
# not elapsed time, because a full-library pass legitimately runs for an hour.
#   LECTIO_SCHEDULER_STALL_SECONDS          no progress for this long => log loudly
#   LECTIO_SCHEDULER_STALL_RESTART_SECONDS  no progress for this long => exit so the
#                                           container restarts; 0 disables
SCHEDULER_STALL_SECONDS = _env_int("LECTIO_SCHEDULER_STALL_SECONDS", 600)
SCHEDULER_STALL_RESTART_SECONDS = _env_int("LECTIO_SCHEDULER_STALL_RESTART_SECONDS", 1800)
SCHEDULER_WATCHDOG_POLL_SECONDS = 60

DEBUG_MODE = os.getenv("LECTIO_DEBUG", "0") == "1"
# Public base URL of this instance (e.g. https://lectio.example.com).
# Required for WebSub: hubs need a reachable callback URL.  Leave blank to disable WebSub.
LECTIO_PUBLIC_URL = os.getenv("LECTIO_PUBLIC_URL", "").strip().rstrip("/")

_FEVER_PASSWORD = os.getenv("LECTIO_FEVER_PASSWORD", "").strip()

# --- Email (Resend) config — env vars are fallbacks; DB settings take precedence at runtime ---
_ENV_RESEND_API_KEY = os.getenv("RESEND_API_KEY", "").strip()
_ENV_RESEND_FROM = os.getenv("LECTIO_EMAIL_FROM", "").strip()
# Seed value for the bootstrap admin's default recipient (then per-user).
_ENV_RESEND_TO = os.getenv("LECTIO_EMAIL_TO", "").strip()


def get_resend_api_key() -> str:
    # Instance-shared: the Resend account/key is owned at the instance level
    # (one verified domain), so the env value is the shared default for everyone.
    return get_runtime_setting(SETTING_RESEND_API_KEY, _ENV_RESEND_API_KEY)


def get_resend_from() -> str:
    # Instance sending identity (admin-managed, like the Resend key): env value is
    # the shared default for everyone; an admin override lives in their settings.
    return get_runtime_setting(SETTING_EMAIL_FROM, _ENV_RESEND_FROM)


def is_email_configured() -> bool:
    return bool(get_resend_api_key() and get_resend_from())


def is_instapaper_configured() -> bool:
    return bool(get_runtime_setting(SETTING_INSTAPAPER_USERNAME) and get_runtime_setting(SETTING_INSTAPAPER_PASSWORD))


# --- YouTube subscription sync config — env vars are fallbacks; DB settings take precedence ---
_ENV_YT_API_KEY = os.getenv("YOUTUBE_API_KEY", "").strip()
_ENV_YT_CHANNEL_ID = os.getenv("YOUTUBE_CHANNEL_ID", "").strip()
_ENV_YT_FOLDER_NAME = os.getenv("YOUTUBE_FOLDER_NAME", "").strip() or "YouTube Subscriptions"
# DeviantArt API creds — per-user DB settings take precedence; env is the fallback.
_ENV_DEVIANTART_CLIENT_ID = os.getenv("DEVIANTART_CLIENT_ID", "").strip()
_ENV_DEVIANTART_CLIENT_SECRET = os.getenv("DEVIANTART_CLIENT_SECRET", "").strip()
# YouTube OAuth client creds are app-level (one registered Google app), not
# per-user — only the resulting tokens are per-user. So these are read straight
# from env for every user (sharing the app, never the tokens).
_ENV_YT_OAUTH_CLIENT_ID = os.getenv("YOUTUBE_OAUTH_CLIENT_ID", "").strip()
_ENV_YT_OAUTH_CLIENT_SECRET = os.getenv("YOUTUBE_OAUTH_CLIENT_SECRET", "").strip()
# Pinterest OAuth client creds are app-level (one registered Pinterest app), same
# as YouTube above — only the resulting tokens are per-user.
_ENV_PINTEREST_OAUTH_CLIENT_ID = os.getenv("PINTEREST_OAUTH_CLIENT_ID", "").strip()
_ENV_PINTEREST_OAUTH_CLIENT_SECRET = os.getenv("PINTEREST_OAUTH_CLIENT_SECRET", "").strip()
# Quire API creds — per-user DB settings take precedence; env is the fallback
# (same pattern as DeviantArt). Only the resulting tokens are ever per-user.
_ENV_QUIRE_CLIENT_ID = os.getenv("QUIRE_CLIENT_ID", "").strip()
_ENV_QUIRE_CLIENT_SECRET = os.getenv("QUIRE_CLIENT_SECRET", "").strip()
# Inoreader migration OAuth creds — env is the fallback; DB settings take precedence.
_ENV_INOREADER_CLIENT_ID = os.getenv("INOREADER_CLIENT_ID", "").strip()
_ENV_INOREADER_CLIENT_SECRET = os.getenv("INOREADER_CLIENT_SECRET", "").strip()
_yt_sync_hour_raw = os.getenv("YOUTUBE_SYNC_HOUR", "").strip()
_ENV_MAINTENANCE_HOUR: int | None = None
if _yt_sync_hour_raw:
    try:
        _h = int(_yt_sync_hour_raw)
        if 0 <= _h <= 23:
            _ENV_MAINTENANCE_HOUR = _h
    except ValueError:
        pass


def get_yt_api_key() -> str:
    return get_runtime_setting(SETTING_YT_API_KEY, _ENV_YT_API_KEY)


def get_yt_channel_id() -> str:
    return get_runtime_setting(SETTING_YT_CHANNEL_ID, _ENV_YT_CHANNEL_ID)


def get_yt_folder_name() -> str:
    return get_runtime_setting(SETTING_YT_FOLDER_NAME, _ENV_YT_FOLDER_NAME) or "YouTube Subscriptions"


def youtube_embed_account_features_enabled() -> bool:
    """Per-user: True if YouTube embeds should use the standard youtube.com host
    (Share / Watch Later) instead of the privacy-enhanced -nocookie host.
    Privacy-enhanced is the default."""
    return get_runtime_setting(SETTING_YT_EMBED_ACCOUNT_FEATURES, "0") == "1"


def youtube_hide_shorts_global() -> bool:
    """Per-user: auto-mark Shorts read on ALL YouTube feeds at refresh (overrides the
    per-feed pref). Off by default."""
    return get_runtime_setting(SETTING_YT_HIDE_SHORTS_GLOBAL, "0") == "1"


def youtube_hide_unpremiered_global() -> bool:
    """Per-user: hide not-yet-premiered videos on ALL YouTube feeds (overrides the
    per-feed pref). Off by default."""
    return get_runtime_setting(SETTING_YT_HIDE_UNPREMIERED_GLOBAL, "0") == "1"


def youtube_hide_members_only_global() -> bool:
    """Per-user: auto-mark members-only videos read on ALL YouTube feeds at
    refresh (overrides the per-feed pref). Off by default."""
    return get_runtime_setting(SETTING_YT_HIDE_MEMBERS_ONLY_GLOBAL, "0") == "1"


def hide_locked_comics_global() -> bool:
    """Per-user: hide a supporter-locked webcomic strip on ALL webcomic feeds
    until its own stated unlock date (overrides the per-feed pref). Off by
    default."""
    return get_runtime_setting(SETTING_HIDE_LOCKED_COMICS_GLOBAL, "0") == "1"


def _pacific_today() -> str:
    """Today's date (YYYY-MM-DD) in US/Pacific — the timezone YouTube resets quota on."""
    import datetime as _dt

    try:
        from zoneinfo import ZoneInfo

        return _dt.datetime.now(ZoneInfo("America/Los_Angeles")).strftime("%Y-%m-%d")
    except Exception:
        # Fallback: fixed -08:00 offset (good enough for a day-bucket if tzdata is absent).
        return (_dt.datetime.now(_dt.timezone.utc) - _dt.timedelta(hours=8)).strftime("%Y-%m-%d")


def youtube_quota_cap() -> int:
    try:
        return max(1, int(get_runtime_setting(SETTING_YT_QUOTA_CAP, str(YT_QUOTA_DEFAULT_CAP)) or YT_QUOTA_DEFAULT_CAP))
    except TypeError, ValueError:
        return YT_QUOTA_DEFAULT_CAP


def record_yt_quota_spend(units: int) -> None:
    """Add ``units`` to the current user's Pacific-day YouTube quota tally. Called by
    the YT services after each billed API call (sink wired in at startup)."""
    if units <= 0:
        return
    try:
        with get_meta_connection() as conn:
            conn.execute(
                "INSERT INTO yt_quota_spend (day, units) VALUES (?, ?) ON CONFLICT(day) DO UPDATE SET units = units + excluded.units",
                (_pacific_today(), int(units)),
            )
    except Exception:
        LOGGER.debug("[yt-quota] failed to record %d units", units, exc_info=True)


def mark_yt_quota_exhausted() -> None:
    """Snap today's tally to the cap after an actual quotaExceeded response."""
    try:
        cap = youtube_quota_cap()
        with get_meta_connection() as conn:
            conn.execute(
                "INSERT INTO yt_quota_spend (day, units) VALUES (?, ?) ON CONFLICT(day) DO UPDATE SET units = MAX(units, excluded.units)",
                (_pacific_today(), cap),
            )
    except Exception:
        LOGGER.debug("[yt-quota] failed to mark exhausted", exc_info=True)


def get_yt_quota_spent_today() -> int:
    try:
        with get_meta_connection() as conn:
            row = conn.execute("SELECT units FROM yt_quota_spend WHERE day = ?", (_pacific_today(),)).fetchone()
        return int(row["units"]) if row else 0
    except Exception:
        return 0


def get_yt_quota_status() -> dict:
    """Quota meter payload for the UI: spent / cap / remaining + a low/exhausted flag."""
    cap = youtube_quota_cap()
    spent = get_yt_quota_spent_today()
    remaining = max(0, cap - spent)
    state = "exhausted" if remaining <= 0 else ("low" if remaining < 500 else "ok")
    return {"spent": spent, "cap": cap, "remaining": remaining, "state": state}


def youtube_embed_host() -> str:
    """The YouTube embed host for the current user's privacy/features preference."""
    return "www.youtube.com" if youtube_embed_account_features_enabled() else "www.youtube-nocookie.com"


def invalidate_instance_setting_cache() -> None:
    with _instance_setting_cache_lock:
        _instance_setting_cache.clear()


def _get_admin_instance_setting(key: str) -> str:
    """First enabled admin's stored value for *key* ("" if none), from any context.

    Reads via get_setting (DB-backed) rather than the in-memory cache alone, so
    it works in background threads before the admin's cache has ever loaded
    (e.g. the 3am maintenance check right after a container restart).
    """
    now = time.monotonic()
    with _instance_setting_cache_lock:
        hit = _instance_setting_cache.get(key)
        if hit and now - hit[0] < _INSTANCE_SETTING_TTL_SECONDS:
            return hit[1]
    val = ""
    # Never crash the caller (login lockout checks run pre-auth; a
    # half-provisioned admin dir must degrade to the env fallback).
    try:
        for u in user_store.list_users():
            if not u.get("is_admin") or u.get("disabled"):
                continue
            with tenancy.user_context(u["user_id"]):
                with get_meta_connection() as conn:
                    v = get_setting(conn, key)
            if v:
                val = v
                break
    except Exception:
        LOGGER.debug("admin instance-setting lookup failed for %r", key, exc_info=True)
    with _instance_setting_cache_lock:
        _instance_setting_cache[key] = (now, val)
    return val


def get_instance_setting(key: str, env_fallback: str = "") -> str:
    """Resolve an instance-level (admin-managed) setting from any context.

    Resolution: current context's own setting (when a user is bound) → first
    enabled admin's setting → env fallback. The admin tier is what makes
    Administration → Instance Config authoritative for background threads,
    pre-auth checks, and other users.
    """
    val = get_cached_setting(key)
    if val:
        return val
    return _get_admin_instance_setting(key) or env_fallback


def _get_shared_credential(key: str) -> str:
    """Read a shared-instance credential from the first admin user's settings.

    Shared-instance OAuth creds are stored in the admin's own app_settings
    under a namespaced key (e.g. shared_yt_oauth_client_id) and are used as a
    middle tier: per-user setting → shared-instance → env var.
    """
    return _get_admin_instance_setting(key)


def get_fetch_history_max_age_days() -> int:
    return int(get_instance_setting(SETTING_FETCH_HISTORY_MAX_AGE_DAYS) or 30)


# Fixed per-feed row ceiling for feed_fetch_history — a backstop against a
# pathologically-retried feed, not a user-facing setting. Normal cadence (even
# hourly for the full max-age window) stays far under this.
FETCH_HISTORY_ROW_CEILING = 5000


def get_tombstone_sweep_days() -> int:
    """Age (days) after which deleted-entry tombstones are swept by nightly
    maintenance; 0 disables sweeping. By then the entry has long left the
    publisher's feed window, so a refresh can no longer resurrect it."""
    return int(get_instance_setting(SETTING_TOMBSTONE_SWEEP_DAYS) or 90)


def get_login_max_failures() -> int:
    return int(get_instance_setting(SETTING_LOGIN_MAX_FAILURES) or 5)


def get_login_window_seconds() -> int:
    return int(get_instance_setting(SETTING_LOGIN_WINDOW_SECONDS) or 300)


def get_instance_default_auto_refresh() -> int:
    raw = int(get_instance_setting(SETTING_DEFAULT_AUTO_REFRESH_MINUTES) or DEFAULT_AUTO_REFRESH_MINUTES)
    return 0 if raw <= 0 else max(raw, MIN_AUTO_REFRESH_MINUTES)


def get_youtube_oauth_credentials() -> tuple[str, str]:
    """YouTube OAuth client (client_id, client_secret).

    Resolution: per-user setting → shared-instance (admin's settings) → env var.
    """
    cid = (
        get_runtime_setting(SETTING_YT_OAUTH_CLIENT_ID)
        or _get_shared_credential(SETTING_SHARED_YT_OAUTH_CLIENT_ID)
        or _ENV_YT_OAUTH_CLIENT_ID
    )
    secret = (
        get_runtime_setting(SETTING_YT_OAUTH_CLIENT_SECRET)
        or _get_shared_credential(SETTING_SHARED_YT_OAUTH_CLIENT_SECRET)
        or _ENV_YT_OAUTH_CLIENT_SECRET
    )
    return cid, secret


def youtube_oauth_connected() -> bool:
    """True if the current user has connected their YouTube account."""
    with get_meta_connection() as conn:
        return bool(get_setting(conn, SETTING_YT_OAUTH_REFRESH_TOKEN))


def get_youtube_oauth_token() -> str:
    """Return a valid user access token for playlist writes, refreshing if needed.

    Empty string if the user hasn't connected their YouTube account or the
    refresh failed (revoked / testing-mode 7-day expiry) — caller must prompt a
    reconnect.
    """
    with get_meta_connection() as conn:
        access = get_setting(conn, SETTING_YT_OAUTH_ACCESS_TOKEN) or ""
        refresh = get_setting(conn, SETTING_YT_OAUTH_REFRESH_TOKEN) or ""
        try:
            expires_at = float(get_setting(conn, SETTING_YT_OAUTH_TOKEN_EXPIRES_AT) or 0)
        except TypeError, ValueError:
            expires_at = 0.0
    if not refresh:
        return ""
    if access and time.time() < expires_at - 60:
        return access
    cid, secret = get_youtube_oauth_credentials()
    try:
        data = youtube_oauth_service.refresh_access_token(cid, secret, refresh)
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("[youtube-oauth] token refresh failed; reconnect required: %s", exc)
        return ""
    with get_meta_connection() as conn:
        set_setting(conn, SETTING_YT_OAUTH_ACCESS_TOKEN, data["access_token"])
        if data.get("refresh_token"):
            set_setting(conn, SETTING_YT_OAUTH_REFRESH_TOKEN, data["refresh_token"])
        set_setting(conn, SETTING_YT_OAUTH_TOKEN_EXPIRES_AT, str(time.time() + float(data.get("expires_in", 3600))))
    return data["access_token"]


def get_pinterest_oauth_credentials() -> tuple[str, str]:
    """Pinterest OAuth client (client_id, client_secret).

    Resolution: per-user setting → shared-instance (admin's settings) → env var.
    """
    cid = (
        get_runtime_setting(SETTING_PINTEREST_OAUTH_CLIENT_ID)
        or _get_shared_credential(SETTING_SHARED_PINTEREST_OAUTH_CLIENT_ID)
        or _ENV_PINTEREST_OAUTH_CLIENT_ID
    )
    secret = (
        get_runtime_setting(SETTING_PINTEREST_OAUTH_CLIENT_SECRET)
        or _get_shared_credential(SETTING_SHARED_PINTEREST_OAUTH_CLIENT_SECRET)
        or _ENV_PINTEREST_OAUTH_CLIENT_SECRET
    )
    return cid, secret


def pinterest_oauth_connected() -> bool:
    """True if the current user has connected their Pinterest account."""
    with get_meta_connection() as conn:
        return bool(get_setting(conn, SETTING_PINTEREST_OAUTH_REFRESH_TOKEN))


def get_pinterest_oauth_token() -> str:
    """Return a valid user access token for pin writes, refreshing if needed.

    Empty string if the user hasn't connected Pinterest or the refresh failed
    (revoked / expired) — caller must prompt a reconnect."""
    with get_meta_connection() as conn:
        access = get_setting(conn, SETTING_PINTEREST_OAUTH_ACCESS_TOKEN) or ""
        refresh = get_setting(conn, SETTING_PINTEREST_OAUTH_REFRESH_TOKEN) or ""
        try:
            expires_at = float(get_setting(conn, SETTING_PINTEREST_OAUTH_TOKEN_EXPIRES_AT) or 0)
        except TypeError, ValueError:
            expires_at = 0.0
    if not refresh:
        return ""
    if access and time.time() < expires_at - 60:
        return access
    cid, secret = get_pinterest_oauth_credentials()
    try:
        data = pinterest_oauth_service.refresh_access_token(cid, secret, refresh)
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("[pinterest-oauth] token refresh failed; reconnect required: %s", exc)
        return ""
    with get_meta_connection() as conn:
        set_setting(conn, SETTING_PINTEREST_OAUTH_ACCESS_TOKEN, data["access_token"])
        if data.get("refresh_token"):
            set_setting(conn, SETTING_PINTEREST_OAUTH_REFRESH_TOKEN, data["refresh_token"])
        set_setting(conn, SETTING_PINTEREST_OAUTH_TOKEN_EXPIRES_AT, str(time.time() + float(data.get("expires_in", 3600))))
    return data["access_token"]


def get_reddit_credentials() -> tuple[str, str]:
    """Reddit OAuth client (client_id, client_secret).

    Resolution: per-user setting → shared-instance (admin's settings) → empty.
    No env-var fallback — credentials are DB-only for this integration.
    """
    cid = get_runtime_setting(SETTING_REDDIT_CLIENT_ID) or _get_shared_credential(SETTING_SHARED_REDDIT_CLIENT_ID) or ""
    secret = get_runtime_setting(SETTING_REDDIT_CLIENT_SECRET) or _get_shared_credential(SETTING_SHARED_REDDIT_CLIENT_SECRET) or ""
    return cid, secret


def reddit_connected() -> bool:
    """True if the current user has connected their Reddit account."""
    with get_meta_connection() as conn:
        return bool(get_setting(conn, SETTING_REDDIT_REFRESH_TOKEN))


def get_reddit_user_token() -> str:
    """Return a valid Reddit access token, refreshing if needed.

    Empty string if not connected or refresh failed — caller must prompt reconnect.
    """
    with get_meta_connection() as conn:
        access = get_setting(conn, SETTING_REDDIT_ACCESS_TOKEN) or ""
        refresh = get_setting(conn, SETTING_REDDIT_REFRESH_TOKEN) or ""
        try:
            expires_at = float(get_setting(conn, SETTING_REDDIT_TOKEN_EXPIRES_AT) or 0)
        except TypeError, ValueError:
            expires_at = 0.0
    if not refresh:
        return ""
    if access and time.time() < expires_at - 60:
        return access
    cid, secret = get_reddit_credentials()
    if not cid or not secret:
        return ""
    try:
        data = reddit_service.refresh_access_token(cid, secret, refresh)
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("[reddit] token refresh failed; reconnect required: %s", exc)
        return ""
    with get_meta_connection() as conn:
        set_setting(conn, SETTING_REDDIT_ACCESS_TOKEN, data["access_token"])
        if data.get("refresh_token"):
            set_setting(conn, SETTING_REDDIT_REFRESH_TOKEN, data["refresh_token"])
        set_setting(conn, SETTING_REDDIT_TOKEN_EXPIRES_AT, str(time.time() + float(data.get("expires_in", 3600))))
    return data["access_token"]


def _reddit_redirect_uri(request: Request) -> str:
    base = os.getenv("LECTIO_PUBLIC_URL", "").rstrip("/")
    if not base:
        base = str(request.base_url).rstrip("/")
    return f"{base}/integrations/reddit/oauth/callback"


def get_inoreader_credentials() -> tuple[str, str]:
    """Inoreader OAuth credentials (client_id, client_secret).

    DB settings take precedence over env vars so any user can override them
    without restarting the container.
    """
    cid = get_runtime_setting(SETTING_INOREADER_CLIENT_ID, _ENV_INOREADER_CLIENT_ID)
    secret = get_runtime_setting(SETTING_INOREADER_CLIENT_SECRET, _ENV_INOREADER_CLIENT_SECRET)
    return cid, secret


def inoreader_connected() -> bool:
    with get_meta_connection() as conn:
        return bool(get_setting(conn, SETTING_INOREADER_REFRESH_TOKEN))


def get_inoreader_token() -> str:
    """Return a valid Inoreader access token, refreshing it if expired."""
    with get_meta_connection() as conn:
        access = get_setting(conn, SETTING_INOREADER_ACCESS_TOKEN) or ""
        refresh = get_setting(conn, SETTING_INOREADER_REFRESH_TOKEN) or ""
        expires_at = float(get_setting(conn, SETTING_INOREADER_TOKEN_EXPIRES_AT) or 0)
    if not refresh:
        return ""
    if access and time.time() < expires_at - 60:
        return access
    cid, secret = get_inoreader_credentials()
    try:
        data = inoreader_service.refresh_access_token(cid, secret, refresh)
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("[inoreader] token refresh failed: %s", exc)
        if "invalid_grant" in str(exc):
            # Inoreader has permanently rejected this refresh token (not a transient
            # network/outage failure) — clear it so inoreader_connected() stops
            # reporting stale "Connected." and the UI offers Connect again.
            with get_meta_connection() as conn:
                set_setting(conn, SETTING_INOREADER_ACCESS_TOKEN, "")
                set_setting(conn, SETTING_INOREADER_REFRESH_TOKEN, "")
                set_setting(conn, SETTING_INOREADER_TOKEN_EXPIRES_AT, "")
        return ""
    with get_meta_connection() as conn:
        set_setting(conn, SETTING_INOREADER_ACCESS_TOKEN, data["access_token"])
        if data.get("refresh_token"):
            set_setting(conn, SETTING_INOREADER_REFRESH_TOKEN, data["refresh_token"])
        set_setting(conn, SETTING_INOREADER_TOKEN_EXPIRES_AT, str(time.time() + float(data.get("expires_in", 3600))))
    return data["access_token"]


def get_deviantart_credentials() -> tuple[str, str]:
    """Per-user DeviantArt API credentials (client_id, client_secret).

    Stored per-user in app-settings; env vars are instance-wide fallbacks.
    """
    cid = get_runtime_setting(SETTING_DEVIANTART_CLIENT_ID, _ENV_DEVIANTART_CLIENT_ID)
    secret = get_runtime_setting(SETTING_DEVIANTART_CLIENT_SECRET, _ENV_DEVIANTART_CLIENT_SECRET)
    return cid, secret


def get_deviantart_user_token() -> str:
    """Return a valid user access token for watch-list ops, refreshing if expired.

    Empty string if the user hasn't connected their DeviantArt account.
    """
    with get_meta_connection() as conn:
        access = get_setting(conn, SETTING_DEVIANTART_ACCESS_TOKEN) or ""
        refresh = get_setting(conn, SETTING_DEVIANTART_REFRESH_TOKEN) or ""
        try:
            expires_at = float(get_setting(conn, SETTING_DEVIANTART_TOKEN_EXPIRES_AT) or 0)
        except TypeError, ValueError:
            # A corrupt/manually-edited value must not break every token read;
            # treat it as expired so we fall through to a forced refresh.
            expires_at = 0.0
    if not access:
        return ""
    if time.time() < expires_at - 60:
        return access
    if not refresh:
        return access  # no refresh token; try the (possibly stale) access token
    cid, secret = get_deviantart_credentials()
    try:
        data = deviantart_service.refresh_access_token(cid, secret, refresh)
    except Exception as exc:  # noqa: BLE001
        # Refresh failed (expired/rotated refresh token, or the client_secret no
        # longer matches the app) — the session is dead; signal "reconnect needed"
        # rather than handing back a stale token that 401s deep in a sync.
        LOGGER.warning("[deviantart] token refresh failed; reconnect required: %s", exc)
        return ""
    with get_meta_connection() as conn:
        set_setting(conn, SETTING_DEVIANTART_ACCESS_TOKEN, data["access_token"])
        if data.get("refresh_token"):
            set_setting(conn, SETTING_DEVIANTART_REFRESH_TOKEN, data["refresh_token"])
        set_setting(conn, SETTING_DEVIANTART_TOKEN_EXPIRES_AT, str(time.time() + float(data.get("expires_in", 3600))))
    return data["access_token"]


def get_quire_credentials() -> tuple[str, str]:
    """Per-user Quire API credentials (client_id, client_secret).

    Stored per-user in app-settings; env vars are instance-wide fallbacks.
    """
    cid = get_runtime_setting(SETTING_QUIRE_CLIENT_ID, _ENV_QUIRE_CLIENT_ID)
    secret = get_runtime_setting(SETTING_QUIRE_CLIENT_SECRET, _ENV_QUIRE_CLIENT_SECRET)
    return cid, secret


def is_quire_connected() -> bool:
    """True once the user has completed the Quire OAuth flow (has a token)."""
    return bool(get_runtime_setting(SETTING_QUIRE_ACCESS_TOKEN))


def quire_project_oid() -> str:
    """The chosen default destination project's OID ("" if none picked yet)."""
    return (get_runtime_setting(SETTING_QUIRE_PROJECT_OID) or "").strip()


def quire_project_name() -> str:
    """Display name of the chosen default destination project ("" if none)."""
    return (get_runtime_setting(SETTING_QUIRE_PROJECT_NAME) or "").strip()


def is_quire_configured() -> bool:
    """True when Quire is connected AND a default destination project is set —
    the precondition for the entry button, On-Star, and Automation rules."""
    return bool(is_quire_connected() and quire_project_oid())


def get_quire_user_token() -> str:
    """Return a valid Quire access token, refreshing if expired. Empty string if the
    user hasn't connected their Quire account."""
    with get_meta_connection() as conn:
        access = get_setting(conn, SETTING_QUIRE_ACCESS_TOKEN) or ""
        refresh = get_setting(conn, SETTING_QUIRE_REFRESH_TOKEN) or ""
        try:
            expires_at = float(get_setting(conn, SETTING_QUIRE_TOKEN_EXPIRES_AT) or 0)
        except TypeError, ValueError:
            expires_at = 0.0
    if not access:
        return ""
    if time.time() < expires_at - 60:
        return access
    if not refresh:
        return access
    cid, secret = get_quire_credentials()
    try:
        data = quire_service.refresh_access_token(cid, secret, refresh)
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("[quire] token refresh failed; reconnect required: %s", exc)
        return ""
    with get_meta_connection() as conn:
        set_setting(conn, SETTING_QUIRE_ACCESS_TOKEN, data["access_token"])
        if data.get("refresh_token"):
            set_setting(conn, SETTING_QUIRE_REFRESH_TOKEN, data["refresh_token"])
        set_setting(conn, SETTING_QUIRE_TOKEN_EXPIRES_AT, str(time.time() + float(data.get("expires_in", 3600))))
    return data["access_token"]


def quire_rate_caps() -> tuple[int, int]:
    """(per-minute, per-hour) caps Lectio meters its Quire calls against."""

    def _cap(key: str, default: int) -> int:
        try:
            return max(1, int(get_runtime_setting(key, str(default)) or default))
        except TypeError, ValueError:
            return default

    return (_cap(SETTING_QUIRE_RATE_CAP_MIN, QUIRE_RATE_DEFAULT_CAP_MIN), _cap(SETTING_QUIRE_RATE_CAP_HOUR, QUIRE_RATE_DEFAULT_CAP_HOUR))


def record_quire_call(calls: int = 1) -> None:
    """Log ``calls`` Quire API calls into the sliding-window meter (wired as the
    service usage sink at startup). Best-effort; prunes rows older than an hour."""
    if calls <= 0:
        return
    try:
        now = int(time.time())
        with get_meta_connection() as conn:
            conn.executemany(
                "INSERT INTO quire_call_log (ts) VALUES (?)",
                [(now,) for _ in range(int(calls))],
            )
            conn.execute("DELETE FROM quire_call_log WHERE ts < ?", (now - 3600,))
    except Exception:
        LOGGER.debug("[quire] failed to record %d calls", calls, exc_info=True)


def get_quire_usage_status() -> dict:
    """Usage-meter payload: calls in the last minute/hour vs caps, plus a state flag."""
    cap_min, cap_hour = quire_rate_caps()
    try:
        now = int(time.time())
        with get_meta_connection() as conn:
            minute_used = conn.execute("SELECT COUNT(*) AS n FROM quire_call_log WHERE ts >= ?", (now - 60,)).fetchone()["n"]
            hour_used = conn.execute("SELECT COUNT(*) AS n FROM quire_call_log WHERE ts >= ?", (now - 3600,)).fetchone()["n"]
    except Exception:
        minute_used = hour_used = 0
    minute_used, hour_used = int(minute_used), int(hour_used)
    blocked = minute_used >= cap_min or hour_used >= cap_hour
    low = (minute_used >= cap_min * 0.8) or (hour_used >= cap_hour * 0.8)
    state = "blocked" if blocked else ("low" if low else "ok")
    return {"minute_used": minute_used, "minute_cap": cap_min, "hour_used": hour_used, "hour_cap": cap_hour, "state": state}


def detect_quire_plan_and_caps() -> str:
    """Detect the destination project's organization plan and align the rate-meter
    caps to it (Free 50/200, Professional 300/1250, Premium 1000/5000). Stores the
    plan name for display. Best-effort; returns the detected plan ("" on failure).

    Quire rate-limits per organization, so the cap that matters is the plan of the
    org owning the chosen project."""
    project_oid = quire_project_oid()
    if not project_oid:
        return ""
    token = get_quire_user_token()
    if not token:
        return ""
    try:
        plan = quire_service.get_project_plan(token, project_oid)
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("[quire] plan detection failed: %s", exc)
        return ""
    caps = quire_service.PLAN_RATE_CAPS.get(plan.strip().lower())
    with get_meta_connection() as conn:
        set_setting(conn, SETTING_QUIRE_PLAN, plan)
        if caps:
            # Enterprise (and any unknown plan) keeps the existing/default caps since
            # its quota scales with member count and isn't a fixed pair.
            set_setting(conn, SETTING_QUIRE_RATE_CAP_MIN, str(caps[0]))
            set_setting(conn, SETTING_QUIRE_RATE_CAP_HOUR, str(caps[1]))
    return plan


def _deviantart_folder_name() -> str:
    return get_runtime_setting(SETTING_DEVIANTART_FOLDER_NAME) or "DeviantArt"


def _apply_deviantart_image_strategy(conn: sqlite3.Connection, file_url: str) -> None:
    """Pin a DeviantArt feed to the 'inline' image strategy.

    DA feeds embed the authoritative full-size image inline (a stable, non-expiring
    wixmp URL), so deriving both the article lead image and the list thumbnail from
    that content — statelessly, every render — avoids source-page scraping and the
    lead-image cache getting clobbered to 'no image'. Locked (manual) so auto-detect
    can't change it.
    """
    try:
        lead_image_service.store_feed_strategy(file_url, "inline", manual=True)
    except Exception:
        LOGGER.exception("[deviantart] failed to pin inline strategy for %s", file_url)
    try:
        conn.execute("INSERT INTO feed_display_prefs (feed_url) VALUES (?) ON CONFLICT(feed_url) DO NOTHING", (file_url,))
        conn.execute("UPDATE feed_display_prefs SET thumb_strategy = 'inline' WHERE feed_url = ?", (file_url,))
    except Exception:
        LOGGER.exception("[deviantart] failed to set inline thumb_strategy for %s", file_url)


def _is_da_deactivated_error(exc: Exception) -> bool:
    """True when a DeviantArt add failed because the artist deactivated their
    account — DA returns HTTP 400 with ``"Account is inactive."``. Distinct from
    a 404 (deleted/renamed), so we can park it for a daily reactivation re-check
    instead of retrying every sync."""
    return "account is inactive" in str(exc).lower()


def _humanize_da_add_error(exc: Exception) -> str:
    """Short, user-facing reason a watch-list add failed, for the Settings list.

    DeviantArt gallery fetches raise ``RuntimeError('gallery fetch failed for X:
    HTTP 404: ...')``; surface the status where we can, else a trimmed message."""
    detail = " ".join(str(exc).split())
    for needle, label in (
        ("Account is inactive", "deactivated"),
        ("404", "not found"),
        ("403", "access denied"),
        ("500", "DeviantArt error"),
    ):
        if needle in detail:
            return label
    return detail[:120] or "unknown error"


def _load_da_sync_detail() -> dict:
    """Parse the structured sync-detail setting, or an empty shell on miss/garbage."""
    raw = get_runtime_setting(SETTING_DEVIANTART_SYNC_DETAIL)
    if raw:
        try:
            data = json.loads(raw)
            if isinstance(data, dict):
                return data
        except ValueError, TypeError:
            pass
    return {"failed": [], "unwatched": []}


def _da_deactivated_usernames(conn: sqlite3.Connection) -> set[str]:
    """Lower-cased usernames of watched artists whose DA account is deactivated."""
    return {str(r["username"]).lower() for r in conn.execute("SELECT username FROM deviantart_deactivated").fetchall()}


def _da_deactivated_list() -> list[dict]:
    """Parked deactivated artists for the Settings UI (opens its own connection)."""
    try:
        with get_meta_connection() as conn:
            return [
                {"username": str(r["username"])}
                for r in conn.execute("SELECT username FROM deviantart_deactivated ORDER BY username").fetchall()
            ]
    except Exception:  # noqa: BLE001 — table missing mid-migration, etc.
        return []


def _record_da_deactivated(conn: sqlite3.Connection, username: str) -> None:
    """Mark an artist deactivated (idempotent; preserves first_seen_at)."""
    now = datetime.now(timezone.utc).isoformat()
    conn.execute(
        "INSERT INTO deviantart_deactivated (username, first_seen_at, last_checked_at)"
        " VALUES (?, ?, ?)"
        " ON CONFLICT(username) DO UPDATE SET last_checked_at = excluded.last_checked_at",
        (username, now, now),
    )


_DA_SYNC_RESUME_FALLBACK_S = 900.0  # no Retry-After header → conservative delay


def _schedule_da_sync_resume(uid: str, delay_s: float, next_round: int) -> None:
    def _resume() -> None:
        try:
            result = sync_deviantart_watchlist(auto_resume_round=next_round)
            if result.get("error"):
                with get_meta_connection() as conn:
                    set_setting(conn, SETTING_DEVIANTART_SYNC_STATUS, f"Sync error: {result['error']}")
        except Exception as exc:
            LOGGER.exception("[deviantart] watchlist auto-resume failed")
            with get_meta_connection() as conn:
                set_setting(conn, SETTING_DEVIANTART_SYNC_STATUS, f"Sync failed: {_humanize_da_add_error(exc)}. Click Sync to retry.")

    timer = threading.Timer(delay_s, _run_in_user_context, args=(uid, _resume))
    timer.daemon = True  # lost on restart; the daily maintenance sync catches up
    timer.start()


def sync_deviantart_watchlist(auto_resume_round: int = 0) -> dict:
    """Add a gallery feed for every artist the user Watches (add-only), and
    pause (disable_feed) any subscribed artist newly dropped from the watch
    list — see the reconcile step in _sync_deviantart_watchlist_locked.

    Skips artists already subscribed (anywhere) so it won't duplicate existing
    DeviantArt feeds. When DeviantArt's rate cap interrupts the run, schedules a
    background continuation (honoring Retry-After) instead of waiting for a
    manual re-click. Returns a result summary dict.
    """
    token = get_deviantart_user_token()
    if not token:
        return {"added": 0, "total": 0, "error": "DeviantArt account not connected."}
    uid = tenancy.current_user_id()
    with _da_sync_lock:
        if uid in _da_sync_active:
            LOGGER.info("[deviantart] watchlist sync already running for %s — skipped", uid)
            return {"added": 0, "total": 0, "skipped": True}
        _da_sync_active.add(uid)
    try:
        return _sync_deviantart_watchlist_locked(token, uid, auto_resume_round)
    finally:
        with _da_sync_lock:
            _da_sync_active.discard(uid)


def _sync_deviantart_watchlist_locked(token: str, uid: str, auto_resume_round: int) -> dict:
    cid, secret = get_deviantart_credentials()
    try:
        username = get_runtime_setting(SETTING_DEVIANTART_USERNAME) or deviantart_service.whoami(token)
        watching = deviantart_service.list_watching(token, username)
    except deviantart_service.DeviantArtRateLimited as exc:
        return {"added": 0, "total": 0, "error": f"DeviantArt rate limit hit — try again in a few minutes. ({exc})"}
    except Exception as exc:  # noqa: BLE001
        return {"added": 0, "total": 0, "error": f"Could not read your watch list: {exc}"}

    folder_name = _deviantart_folder_name()
    with get_meta_connection() as conn:
        # Exclude the combined "deviantsyouwatch" Watch feed (source='watch'): its
        # username is a synthetic browse endpoint, not a real artist, so it must
        # not count as a subscribed gallery for add-dedup or the unwatched reconcile
        # (otherwise it's perpetually reported as "subscribed but no longer watched").
        existing = {
            str(r["username"]).lower()
            for r in conn.execute("SELECT username FROM deviantart_feeds WHERE COALESCE(source, 'gallery') != 'watch'").fetchall()
        }
        # Known-deactivated artists can never be added; skip them here so the sync
        # stops re-probing (and re-failing) them every run. Daily maintenance
        # re-checks them for reactivation.
        deactivated = _da_deactivated_usernames(conn)
        folder_id = _get_or_create_folder_by_name(conn, folder_name)
        # The Watch feed IS the watch list — one feed carrying every watched
        # artist's posts. While it exists, a per-artist gallery feed is a
        # duplicate of what it already delivers.
        has_watch_feed = bool(conn.execute("SELECT 1 FROM deviantart_feeds WHERE COALESCE(source, 'gallery') = 'watch' LIMIT 1").fetchone())

    to_add = [a for a in watching if a.lower() not in existing and a.lower() not in deactivated]
    if has_watch_feed and to_add:
        # This sync and the gallery consolidation were undoing each other. The
        # consolidation folds a gallery feed into the Watch feed and then DELETES
        # its deviantart_feeds row (it must — otherwise the refresh loop
        # regenerates the subscription it just merged). That row is this sync's
        # only record of "already subscribed", so every consolidated artist came
        # back looking new and was re-created the next night: 537 gallery feeds
        # returned over two runs after the 2026-08-12 consolidation, filed back
        # into the folder, carrying entries the Watch feed had already delivered.
        LOGGER.info(
            "[deviantart] watchlist sync: Watch feed present — skipping %d per-artist gallery feed(s); the Watch feed already covers them",
            len(to_add),
        )
        to_add = []
    LOGGER.info("[deviantart] watchlist sync: %d watched, %d to add into %r", len(watching), len(to_add), folder_name)
    added = 0
    failed = 0
    deactivated_this_run = 0
    failed_artists: list[dict[str, str]] = []
    rate_limited = False
    retry_after_s: float | None = None
    for i, artist in enumerate(to_add, 1):
        if i > 1:
            time.sleep(0.4)  # pace requests to stay under DeviantArt's rate limit
        try:
            with get_meta_connection() as conn:
                with get_reader() as reader:
                    _fid, file_url = deviantart_service.create_deviantart_feed(
                        conn, reader, artist, cid, secret, access_token=token, limit=24
                    )
                conn.execute(
                    "INSERT OR IGNORE INTO folder_feeds (folder_id, feed_url) VALUES (?, ?)",
                    (folder_id, file_url),
                )
            added += 1
        except deviantart_service.DeviantArtRateLimited as exc:
            # Quota exhausted — stop now (the rest are re-runnable; sync is add-only
            # and dedupes, so the auto-resume continuation picks up where this left off).
            rate_limited = True
            retry_after_s = exc.retry_after
            LOGGER.info("[deviantart] watchlist sync paused at %d/%d (added=%d) — rate limited", i, len(to_add), added)
            break
        except Exception as exc:  # noqa: BLE001
            if _is_da_deactivated_error(exc):
                # Deactivated account — park it (not a hard failure); daily
                # maintenance re-checks for reactivation.
                deactivated_this_run += 1
                with get_meta_connection() as conn:
                    _record_da_deactivated(conn, artist)
                LOGGER.info("[deviantart] watched artist %s is deactivated — parked for re-check", artist)
            else:
                failed += 1
                failed_artists.append({"username": artist, "error": _humanize_da_add_error(exc)})
                LOGGER.warning("[deviantart] watchlist add failed for %s: %s", artist, exc)
        if i % 25 == 0:
            LOGGER.info("[deviantart] watchlist sync progress: %d/%d (added=%d failed=%d)", i, len(to_add), added, failed)
            with get_meta_connection() as conn:
                set_setting(conn, SETTING_DEVIANTART_SYNC_STATUS, f"Syncing {i}/{len(to_add)} (added {added})")
            invalidate_meta_structure_cache()
            invalidate_unread_counts_cache()
    invalidate_meta_structure_cache()
    invalidate_unread_counts_cache()

    # Reconcile: surface subscribed DA artists no longer on the watch list.
    # Unsubscribing stays a manual decision (curated feeds may outlive a
    # Watch), but leaving them actively refreshing was surprising once
    # noticed — so a NEWLY-unwatched artist's feed(s) get paused (disable_feed,
    # same as Settings' own pause/disable — no data lost, nothing unsubscribed)
    # once, the first run that sees them drop off the watch list. Not reapplied
    # on later runs: if Josh re-enables one by hand (decided to keep it live
    # despite the unwatch), a later sync must not fight that choice by
    # disabling it again every time it still shows up as unwatched.
    watching_lower = {a.lower() for a in watching}
    unwatched = sorted(u for u in existing if u not in watching_lower) if watching else []
    prior_unwatched_lower = {str(u.get("username") or "").strip().lower() for u in _load_da_sync_detail().get("unwatched", [])}
    newly_unwatched = [u for u in unwatched if u not in prior_unwatched_lower]
    if newly_unwatched:
        with get_meta_connection() as conn:
            placeholders = ",".join("?" for _ in newly_unwatched)
            rows = conn.execute(
                f"SELECT id, username FROM deviantart_feeds WHERE COALESCE(source, 'gallery') != 'watch'"
                f" AND lower(username) IN ({placeholders})",
                newly_unwatched,
            ).fetchall()
        for row in rows:
            disable_feed(deviantart_service.feed_file_url(str(row["id"])), reason="unwatched on DeviantArt")
        LOGGER.info(
            "[deviantart] %d newly-unwatched artist(s) paused (updates disabled, not unsubscribed): %s",
            len(rows),
            ", ".join(sorted(str(r["username"]) for r in rows)),
        )
        with get_meta_connection() as conn:
            set_setting(conn, SETTING_DEVIANTART_UNWATCHED_DIRTY, "1")
    if unwatched:
        LOGGER.info("[deviantart] %d subscribed artist(s) no longer watched: %s", len(unwatched), ", ".join(unwatched))

    # Persist the failed/unwatched artists (structured) so the Settings UI can
    # list them as profile links. A fresh sync (round 0) starts clean; auto-resume
    # continuation rounds accumulate failures from earlier rounds (each round
    # processes a different slice of to_add). unwatched is recomputed from full
    # data every round, so it's overwritten rather than merged.
    prior_failed: list[dict[str, str]] = []
    if auto_resume_round > 0:
        prior_failed = _load_da_sync_detail().get("failed", [])
    all_failed = prior_failed + failed_artists
    with get_meta_connection() as conn:
        set_setting(
            conn, SETTING_DEVIANTART_SYNC_DETAIL, json.dumps({"failed": all_failed, "unwatched": [{"username": u} for u in unwatched]})
        )

    remaining = len(to_add) - added - failed
    if rate_limited:
        if auto_resume_round < _DA_SYNC_MAX_AUTO_RESUMES:
            delay_s = max(60.0, retry_after_s or _DA_SYNC_RESUME_FALLBACK_S)
            _schedule_da_sync_resume(uid, delay_s, auto_resume_round + 1)
            final = (
                f"Rate limited — added {added} so far, ~{remaining} left. "
                f"Auto-resuming in ~{int(delay_s // 60) or 1} min "
                f"(round {auto_resume_round + 1}/{_DA_SYNC_MAX_AUTO_RESUMES})."
            )
        else:
            final = f"Rate limited — added {added} so far, ~{remaining} left. Auto-resume limit reached; click Sync to continue."
    else:
        # to_add now excludes both already-subscribed and known-deactivated, so
        # derive each count from the watch list directly rather than the residual.
        already = sum(1 for a in watching if a.lower() in existing)
        deactivated_total = sum(1 for a in watching if a.lower() in deactivated) + deactivated_this_run
        parts = [f"Added {added} of {len(watching)} watched"]
        if already:
            parts.append(f"{already} already subscribed")
        if all_failed:
            parts.append(f"{len(all_failed)} failed")
        if deactivated_total:
            parts.append(f"{deactivated_total} deactivated")
        if unwatched:
            parts.append(f"{len(unwatched)} subscribed but no longer watched")
        final = ", ".join(parts)
    with get_meta_connection() as conn:
        set_setting(conn, SETTING_DEVIANTART_SYNC_STATUS, final)
    return {
        "added": added,
        "failed": failed,
        "deactivated": deactivated_this_run,
        "total": len(watching),
        "folder": folder_name,
        "rate_limited": rate_limited,
        "unwatched": unwatched,
        "failed_artists": all_failed,
    }


def _deviantart_recheck_deactivated(max_checks: int = 40) -> int:
    """Re-probe parked deactivated artists; subscribe any that reactivated.

    Runs in daily maintenance. For each parked artist, a lightweight gallery
    fetch tells us the current state: success → reactivated, so create the feed
    and drop it from the deactivated table; still inactive → bump last_checked_at;
    rate-limited → stop early (re-check resumes next maintenance run). Capped per
    run to stay polite against the API quota. Returns the count reactivated."""
    token = get_deviantart_user_token()
    if not token:
        return 0
    cid, secret = get_deviantart_credentials()
    with get_meta_connection() as conn:
        rows = conn.execute(
            "SELECT username FROM deviantart_deactivated ORDER BY last_checked_at ASC LIMIT ?",
            (max_checks,),
        ).fetchall()
    if not rows:
        return 0
    folder_id = None
    reactivated = 0
    now = datetime.now(timezone.utc).isoformat()
    for row in rows:
        artist = str(row["username"])
        try:
            with get_meta_connection() as conn:
                with get_reader() as reader:
                    if folder_id is None:
                        folder_id = _get_or_create_folder_by_name(conn, _deviantart_folder_name())
                    _fid, file_url = deviantart_service.create_deviantart_feed(
                        conn, reader, artist, cid, secret, access_token=token, limit=24
                    )
                conn.execute(
                    "INSERT OR IGNORE INTO folder_feeds (folder_id, feed_url) VALUES (?, ?)",
                    (folder_id, file_url),
                )
                conn.execute("DELETE FROM deviantart_deactivated WHERE username = ?", (artist,))
            reactivated += 1
            LOGGER.info("[deviantart] deactivated artist %s reactivated — subscribed", artist)
        except deviantart_service.DeviantArtRateLimited:
            LOGGER.info("[deviantart] deactivated re-check hit rate limit; stopping this run")
            break
        except Exception as exc:  # noqa: BLE001
            if _is_da_deactivated_error(exc):
                with get_meta_connection() as conn:
                    conn.execute(
                        "UPDATE deviantart_deactivated SET last_checked_at = ? WHERE username = ?",
                        (now, artist),
                    )
            else:
                # A different error (deleted/renamed) — leave it parked; not worth
                # promoting to a hard failure from a background re-check.
                LOGGER.info("[deviantart] deactivated re-check for %s errored (non-inactive): %s", artist, exc)
    if reactivated:
        invalidate_meta_structure_cache()
        invalidate_unread_counts_cache()
    return reactivated


def push_galleries_to_deviantart_watchlist() -> dict:
    """Watch (on DeviantArt) every artist you have a gallery feed for. Add-only."""
    token = get_deviantart_user_token()
    if not token:
        return {"watched": 0, "total": 0, "error": "DeviantArt account not connected."}
    with get_meta_connection() as conn:
        usernames = [str(r["username"]) for r in conn.execute("SELECT DISTINCT username FROM deviantart_feeds").fetchall()]
    watched = 0
    errors = 0
    for name in usernames:
        ok, _msg = deviantart_service.watch_user(token, name)
        if ok:
            watched += 1
        else:
            errors += 1
    return {"watched": watched, "total": len(usernames), "errors": errors}


def get_maintenance_hour() -> int | None:
    val = get_instance_setting(SETTING_MAINTENANCE_HOUR, "")
    if val:
        try:
            h = int(val)
            if 0 <= h <= 23:
                return h
        except ValueError:
            pass
    return _ENV_MAINTENANCE_HOUR


_busy_work_lock = threading.Lock()
_busy_work: dict[str, dict[str, int]] = {}


@contextlib.contextmanager
def busy_work(label: str):
    """Mark the current tenancy user as running long background work (also usable as a decorator).

    Admin "Full vacuum" refuses to touch a user's databases while any of these is open; see user_busy_reasons."""
    uid = tenancy.current_user_id()
    with _busy_work_lock:
        counts = _busy_work.setdefault(uid, {})
        counts[label] = counts.get(label, 0) + 1
    try:
        yield
    finally:
        with _busy_work_lock:
            counts = _busy_work.get(uid, {})
            counts[label] = counts.get(label, 1) - 1
            if counts[label] <= 0:
                counts.pop(label, None)
            if not counts:
                _busy_work.pop(uid, None)


def user_busy_reasons(uid: str, *, include_activity: bool = True, ignore: tuple[str, ...] = ()) -> list[str]:
    """Why a full vacuum of *uid*'s databases should wait, as short phrases; empty means it is safe to go ahead.

    Background work is read from the busy_work marks plus the existing per-user job registries (those need the user's own tenancy
    context). include_activity=False skips the "recently active" check, for an admin vacuuming their own account. Web, Google
    Reader-style and session requests touch last_seen_at; Fever and Miniflux API clients do not, so they can read as idle."""
    reasons: list[str] = []
    if include_activity:
        row = user_store.get_by_id(uid)
        last_seen = float(row["last_seen_at"] or 0) if row else 0.0
        # last_seen_at is written at most once per throttle window, so a user active a second ago can read as up to a window old.
        if time.time() - last_seen < 2 * _LAST_SEEN_THROTTLE_SECONDS:
            reasons.append("the user was active in the last 10 minutes")
    with _busy_work_lock:
        reasons += [f"{label} is running" for label in _busy_work.get(uid, {}) if label not in ignore]
    with tenancy.user_context(uid):
        with _da_sync_lock:
            if uid in _da_sync_active:
                reasons.append("a DeviantArt sync is running")
        if uid in _escalation_reprobe_running:
            reasons.append("an escalation reprobe is running")
        for name, jobs in (
            ("a bulk re-fetch", _refetch_jobs),
            ("a folder refresh", _folder_refresh_jobs),
        ):
            job = jobs.get("job")
            if isinstance(job, dict) and job.get("running"):
                reasons.append(f"{name} is running")
        with _autofetch_jobs_lock:
            # Keyed by (feed_url, entry_id), one job per post, so look through the values rather than for a single "job".
            if any(isinstance(j, dict) and j.get("running") for j in _autofetch_jobs.values()):
                reasons.append("an auto-fetch is running")
    return reasons


def truncate_wal(conn: sqlite3.Connection, label: str, *, attempts: int = 10, delay: float = 1.0) -> bool:
    """Fold the WAL back and truncate it, retrying while live readers block it.

    ``PRAGMA wal_checkpoint(TRUNCATE)`` does not raise when a reader holds the WAL: it returns ``busy=1`` and leaves the file alone.
    After a VACUUM (which writes the whole rebuilt DB through the WAL) that left the WAL at ~DB size, so the total came out larger
    than before. Pooled connections only read briefly, so a short retry usually gets through; if not, say so instead of staying silent.
    """
    # A busy checkpoint waits out the connection's whole busy timeout (30 s on the admin path) before it reports busy, and holds off other
    # writers while it waits; probe quickly instead so the retries take attempts * (0.25 s + delay) and nobody else is locked out for long.
    original_timeout = conn.execute("PRAGMA busy_timeout").fetchone()[0]
    conn.execute("PRAGMA busy_timeout=250")
    try:
        for attempt in range(attempts):
            busy = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()[0]
            if not busy:
                return True
            if attempt + 1 < attempts:
                time.sleep(delay)
    finally:
        conn.execute(f"PRAGMA busy_timeout={int(original_timeout)}")
    LOGGER.warning("[maintenance] %s: WAL not truncated (readers still active); it shrinks at the next checkpoint", label)
    return False


def get_portrait_img_max_width() -> int:
    """Max width (px) for portrait (taller-than-wide) images in article bodies;
    landscape images stay full container width. 0 = disabled. Per-user DB
    override takes precedence over the env fallback."""
    with get_meta_connection() as conn:
        val = get_setting(conn, SETTING_PORTRAIT_IMG_MAX_WIDTH)
    if val:
        try:
            w = int(val)
            if w >= 0:
                return w
        except ValueError:
            pass
    return _ENV_PORTRAIT_IMG_MAX_WIDTH


def mark_read_on_new_tab_enabled() -> bool:
    """Per-user: opening a post's article in a new tab from its list title also marks it read. Off by default."""
    return get_runtime_setting(SETTING_MARK_READ_ON_NEW_TAB, "0") == "1"


def proxy_body_images_enabled() -> bool:
    """Per-user: preemptively proxy every article-pane body image through
    /api/img instead of loading third-party URLs directly. Off by default."""
    return get_runtime_setting(SETTING_PROXY_BODY_IMAGES, "0") == "1"


_PROXY_MODES = ("off", "as_needed", "always")


def get_proxy_url() -> str:
    """Outbound-fetch proxy endpoint (e.g. gluetun's socks5h://gluetun:1080).
    Admin-managed; env default, DB override takes precedence."""
    return get_instance_setting(SETTING_PROXY_URL, _ENV_PROXY_URL)


def get_proxy_mode() -> str:
    """off / as_needed / always for outbound feed fetches. Per-user DB setting
    overrides the instance (first-admin) default; unset resolves to "off" —
    the proxy is opt-in everywhere."""
    val = get_instance_setting(SETTING_PROXY_MODE, "off")
    return val if val in _PROXY_MODES else "off"


def get_tailscale_url() -> str:
    """Last-resort outbound proxy endpoint (e.g. Tailscale's
    socks5h://tailscale:1080). Admin-managed; env default, DB override takes
    precedence. Empty means not configured, which is itself the opt-in gate —
    as_needed escalation never reaches this tier without one."""
    return get_instance_setting(SETTING_TAILSCALE_URL, _ENV_TAILSCALE_URL)


def get_flaresolverr_url() -> str:
    """FlareSolverr endpoint (e.g. http://flaresolverr:8191/v1). Admin-managed;
    env default, DB override takes precedence. Empty means not configured,
    same opt-in gate as the other two backends."""
    return get_instance_setting(SETTING_FLARESOLVERR_URL, _ENV_FLARESOLVERR_URL)


def get_img_cache_days() -> int:
    """Last-accessed TTL (days) for the /api/img cache. 0 = keep forever.
    DB override (admin-managed) takes precedence over the env fallback."""
    val = get_instance_setting(SETTING_IMG_CACHE_DAYS, "")
    if val:
        try:
            d = int(val)
            if d >= 0:
                return d
        except ValueError:
            pass
    return _ENV_IMG_CACHE_DAYS


def get_img_cache_max_dim() -> int:
    """Longest-side px to downscale cached images to. 0 = store originals as-is."""
    val = get_instance_setting(SETTING_IMG_CACHE_MAX_DIM, "")
    if val:
        try:
            d = int(val)
            if d >= 0:
                return d
        except ValueError:
            pass
    return _ENV_IMG_CACHE_MAX_DIM


def get_img_target_bytes() -> int:
    """Byte budget for a cached image, above which it is re-encoded to WebP.

    Separate from the max dimension because the two answer different questions:
    an image can sit exactly at the dimension cap and still weigh 12 MB (a 4K
    RGBA PNG does), and it is the bytes that make it slow to load.
    0 = leave every image at whatever size it arrives.
    """
    val = get_instance_setting(SETTING_IMG_TARGET_BYTES, "")
    if val:
        try:
            n = int(val)
            if n >= 0:
                return n
        except ValueError:
            pass
    return _ENV_IMG_TARGET_BYTES


# --- Auth config ---
# When set, skip the login form and auto-authenticate as the admin on every request.
# Intended for local/private-network installs that don't need a password prompt.
AUTO_LOGIN = os.getenv("LECTIO_AUTO_LOGIN", "0") == "1"

# Password hashing scheme for stored credentials. scrypt and pbkdf2_sha256 are stdlib;
# argon2 needs the optional argon2-cffi package.
PASSWORD_HASH_SCHEME = os.getenv("LECTIO_PASSWORD_HASH_SCHEME", passwords.DEFAULT_SCHEME).strip().lower()
if PASSWORD_HASH_SCHEME not in passwords.available_schemes():
    LOGGER.warning(
        "LECTIO_PASSWORD_HASH_SCHEME=%r not available (have: %s); using %s.",
        PASSWORD_HASH_SCHEME,
        ", ".join(passwords.available_schemes()),
        passwords.DEFAULT_SCHEME,
    )
    PASSWORD_HASH_SCHEME = passwords.DEFAULT_SCHEME

# Bootstrap admin account, seeded on first startup when the users table is empty.
# CHANGE the default password before exposing the instance.
_DEFAULT_ADMIN_PASSWORD = "ChangeA$ap"
BOOTSTRAP_ADMIN_USERNAME = os.getenv("LECTIO_ADMIN_USERNAME", "admin")
BOOTSTRAP_ADMIN_PASSWORD = os.getenv("LECTIO_ADMIN_PASSWORD", _DEFAULT_ADMIN_PASSWORD)

AUTH_ENABLED = True  # kept as a module-level bool so tests can monkeypatch it
user_store: UserStore = UserStore(AUTH_DB_PATH)
SESSION_SECRET_KEY = os.getenv("LECTIO_SECRET_KEY") or secrets.token_hex(32)
if AUTH_ENABLED and not os.getenv("LECTIO_SECRET_KEY"):
    LOGGER.warning(
        "LECTIO_SECRET_KEY is not set — using a random key. Sessions will not survive server restarts. Set a stable key in your .env."
    )
SESSION_MAX_AGE_SECONDS = 365 * 24 * 3600  # 1 year; baked into SessionMiddleware at startup
# Set LECTIO_HTTPS_ONLY=1 when running behind a TLS-terminating reverse proxy.
_HTTPS_ONLY = os.getenv("LECTIO_HTTPS_ONLY", "0") == "1"
# Proxies trusted for X-Forwarded-* headers. "*" (default) trusts any upstream —
# fine when the app port is only reachable via your reverse proxy. Set a
# comma-separated allowlist (e.g. the proxy's IP) for a stricter posture.
_TRUSTED_PROXIES = os.getenv("LECTIO_TRUSTED_PROXIES", "*").strip()
# Have the app emit baseline security headers itself. Default off because a
# reverse proxy (e.g. the bundled Traefik config) usually sets them; enable when
# fronting Lectio with a proxy that does not (Caddy without a header block, a
# bare/no-proxy setup, etc.). Keeps the headers from depending on the proxy.
_SECURITY_HEADERS_ENABLED = os.getenv("LECTIO_SECURITY_HEADERS", "0") == "1"
# Paths that are always public (no login required)
_AUTH_EXEMPT_PREFIXES = (
    "/login",
    "/static",
    "/healthz",
    "/api/img",
    "/api/favicon",
    "/api/save",
    "/api/bookmarklet/save",
    "/dev/feeds/",
    "/fever",
    "/greader/",
    "/v1/",
    "/websub/",
    "/sw.js",
)


def _note_scheduler_progress(stage: str) -> None:
    """Record that the scheduled refresh advanced, and what it is doing now."""
    with _scheduler_state_lock:
        _scheduler_state["last_progress_at"] = time.monotonic()
        _scheduler_state["stage"] = stage


def _scheduler_stall_seconds() -> float | None:
    """Seconds since the in-flight pass last advanced, or None when idle.

    None while no pass is running: between passes there is nothing to stall.
    """
    with _scheduler_state_lock:
        started = _scheduler_state.get("pass_started_at")
        last = _scheduler_state.get("last_progress_at")
    if started is None or last is None:
        return None
    return time.monotonic() - float(last)  # ty: ignore[invalid-argument-type]


def invalidate_meta_structure_cache() -> None:
    with _meta_structure_lock:
        _meta_structure_cache.clear()
    # Refresh-contention investigation (Plan.md Tier 1): this cache is designed
    # to invalidate only on explicit user mutations (subscribe/unsubscribe,
    # folder changes, feed disable/enable) -- a 2026-09-03 live capture showed
    # it going cold roughly every 30-45s during ordinary browsing (no such
    # action taken), each miss costing 5+ real seconds of get_meta_structure_snapshot's
    # 5-query rebuild under refresh contention. There are 37 call sites; rather
    # than audit them all, log the caller so the next occurrence says which one
    # actually fired during a routine refresh pass.
    try:
        caller = sys._getframe(1)
        LOGGER.info(
            "[perf] invalidate_meta_structure_cache from %s:%d (%s)", caller.f_code.co_filename, caller.f_lineno, caller.f_code.co_name
        )
    except Exception:
        pass


def invalidate_problematic_feeds_cache() -> None:
    with _problematic_feeds_cache_lock:
        _problematic_feeds_cache.clear()


def get_problematic_feeds_cached(conn: sqlite3.Connection, limit: int = PROBLEMATIC_FEEDS_LIMIT) -> list[dict]:
    """TTL-cached problematic-feeds list (see cache comment above)."""
    now_pf = time.time()
    with _problematic_feeds_cache_lock:
        cached_pf = _problematic_feeds_cache.get(limit)
    if cached_pf and now_pf - cached_pf[0] < PROBLEMATIC_FEEDS_CACHE_TTL_SECONDS:
        return [dict(r) for r in cached_pf[1]]
    problematic_feeds = feed_refresh_service.get_problematic_feeds(conn, limit=limit)
    with _problematic_feeds_cache_lock:
        _problematic_feeds_cache[limit] = (time.time(), [dict(r) for r in problematic_feeds])
    return [dict(r) for r in problematic_feeds]


def is_async_action_request(request: Request, expected_header: str | None = None) -> bool:
    requested_with = (request.headers.get("X-Requested-With") or "").strip()
    if not requested_with:
        return False
    if expected_header is None:
        return requested_with.startswith("lectio-")
    return requested_with == expected_header


@asynccontextmanager
async def lifespan(app: FastAPI):
    # Startup
    _attach_pending_access_filter()
    ensure_thumb_schema()
    ensure_img_cache_schema()
    ensure_yt_duration_schema()
    ensure_websub_schema()
    bootstrap_admin()

    # Bring every existing user's meta/starred schema up to current. Per-user DBs
    # are otherwise schema-init'd only at provision time, so any table added
    # after a user was provisioned (e.g. feed_fetch_history) is missing from
    # their DB and surfaces as a "no such table" 500 (Feed Properties, etc.).
    # ensure_*_schema are idempotent, so this is a cheap no-op once migrated.
    def _ensure_user_schema() -> None:
        ensure_meta_schema()
        ensure_starred_archive_schema()
        ensure_reader_indexes()
        migrate_spaced_manual_tags()

    _for_each_background_user("per-user schema migration", _ensure_user_schema)

    # Kill switch for heavy startup work. Set LECTIO_DISABLE_STARTUP_BACKFILL=1
    # to skip the scheduled-refresh loop, the FTS index build, and the
    # lead-image / YouTube / archive / read-history backfills. Useful after an
    # OPML import of hundreds of feeds (boot calm, let pages render, then unset
    # and restart). The test suite also sets it so these background DB writers
    # don't race a test's own DB ops on the same temp DB ("database is locked").
    backfill_disabled = os.getenv("LECTIO_DISABLE_STARTUP_BACKFILL", "0") == "1"
    if backfill_disabled:
        LOGGER.warning("LECTIO_DISABLE_STARTUP_BACKFILL=1 — skipping FTS build, scheduled refresh, and backfill threads")

    # reader's FTS index is no longer built or maintained: both search surfaces
    # resolve in SQL (_search_entry_keys_in_sql, _filter_star_keys_by_search)
    # because search_entries builds a highlighted snippet per result, which
    # measured at ~95% of a 10-20s search. Keeping the index cost 1.3ms per new
    # entry on every refresh plus a file about the size of the reader DB itself
    # (564MB against 743MB on the live library) — all of it for an index no
    # query touched. scripts/drop_search_index.py reclaims it on an existing
    # install; a fresh one never creates it.

    if LECTIO_PUBLIC_URL:
        _migrate_websub_to_shared()

    def _prepare_user_dbs() -> None:
        # Checkpoint both WAL files so the first request does not have to rebuild
        # the WAL index from a large file left over from the previous run. Use
        # direct connections so we're not racing the reader.
        for _ckpt_path in (tenancy.reader_db_path(), tenancy.meta_db_path()):
            try:
                with contextlib.closing(sqlite3.connect(str(_ckpt_path), timeout=5)) as _ckpt_conn:
                    _ckpt_conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
            except Exception:
                pass
        with get_meta_connection() as conn:
            purge_lower_level_folders(conn)
            with _app_settings_cache_lock:
                _app_settings_cache[tenancy.current_user_id()] = _load_app_settings_cache(conn)
        with get_reader():  # creates the reader DB if missing
            pass

    _for_each_background_user("per-user DB prep", _prepare_user_dbs)
    app.state.last_scheduled_refresh_started_at = time.monotonic()

    # Warm YouTube duration in-memory cache from DB so first renders are instant.
    youtube_duration_service.warm_cache_from_db()

    # Warm lead image cache from DB so thumbnails are available on first render.
    # Must run per-user: lead images live in each tenant's own meta DB, and the
    # render path (get_cached_entry_thumbnail) only consults the in-memory cache
    # with no per-user DB fallback, so every user's cache must be warmed or their
    # thumbnails stay blank until the rate-limited background backfill catches up.
    _for_each_background_user("lead-image cache warm", lead_image_service.warm_cache_from_db)

    # Warm the per-user unread-counts cache so the first home load after a restart
    # doesn't pay the synchronous reader-DB scan under startup contention (cold
    # cache + concurrent first requests + a refresh kicking off was producing
    # multi-second — occasionally ~20s — home loads right after a redeploy).
    def _warm_unread_counts() -> None:
        try:
            counts = _compute_unread_counts_by_feed()
            with unread_counts_cache_lock:
                unread_counts_cache["unread_counts"] = (time.time(), counts)
        except Exception:
            LOGGER.debug("unread-counts cache warm failed", exc_info=True)

    _for_each_background_user("unread-counts cache warm", _warm_unread_counts)

    # Ensure existing scraped file:// feeds have their entries imported into the
    # reader DB. Feeds may be in backoff from stale "no retriever" errors that
    # pre-date the feed_root='' fix — clear that state and force a reader sync.
    def _sync_scraped_feeds() -> None:
        try:
            with get_meta_connection() as _sc:
                _sf_rows = _sc.execute("SELECT id, mode FROM scraped_feeds").fetchall()
            for _sf_row in _sf_rows:
                _sf_id = str(_sf_row["id"])
                _sf_url = scraper_service.feed_file_url(_sf_id)
                try:
                    with get_meta_connection() as _sc:
                        _sc.execute("DELETE FROM feed_failure_state WHERE feed_url = ?", (_sf_url,))
                        # link_list feeds seed initial links as hidden so only new links
                        # appear as entries.  If ALL entries are still hidden (nothing has
                        # appeared yet), unhide them so the feed isn't permanently empty.
                        if str(_sf_row["mode"]) == "link_list":
                            _visible = _sc.execute(
                                "SELECT count(*) FROM scraped_entries WHERE scraped_feed_id=? AND NOT hidden",
                                (_sf_id,),
                            ).fetchone()[0]
                            if _visible == 0:
                                _sc.execute(
                                    "UPDATE scraped_entries SET hidden=0 WHERE scraped_feed_id=?",
                                    (_sf_id,),
                                )
                                scraper_service._write_feed_file(_sc, _sf_id)
                    with get_reader() as _r:
                        _r.update_feed(_sf_url)
                except Exception:
                    pass
        except Exception:
            LOGGER.exception("[scraper] startup scraped-feed sync failed")

    if not backfill_disabled:
        threading.Thread(
            target=lambda: _for_each_background_user("scraped-feed sync", _sync_scraped_feeds),
            daemon=True,
            name="sync-scraped-feeds",
        ).start()

    # Auto-taggers and the dedup cleanup write per-user tag/strategy state, so
    # run them once per background user (each under its own tenancy context).
    def _startup_auto_tag_and_dedup() -> None:
        # Artwork tagger runs first so webcomic tagger won't clobber ArtStation
        # feeds that live in folders whose name also contains "comic".
        _auto_tag_artwork_feeds()
        # Auto-tag feeds in "comic*" folders with strategy='webcomic'.
        _auto_tag_webcomic_feeds()
        # GitHub release feeds get og_scrape + no list thumbnail.
        _auto_tag_github_release_feeds()

        # Retroactive dedup: suppress duplicate unread entries (same slug or
        # title+date) that accumulated before _suppress_guid_churn could catch them.
        try:
            with get_reader() as reader:
                with get_meta_connection() as conn:
                    cleaned = _cleanup_intra_feed_slug_dupes(reader, conn)
            if cleaned:
                LOGGER.info("[guid-churn-cleanup] suppressed %d pre-existing duplicate entries", cleaned)
        except Exception:
            LOGGER.exception("[guid-churn-cleanup] startup cleanup failed")

    _for_each_background_user("auto-tag/dedup", _startup_auto_tag_and_dedup)

    stop_event = threading.Event()
    if not backfill_disabled:
        thread = threading.Thread(
            target=scheduled_refresh_loop,
            args=(stop_event,),
            daemon=True,
        )
        app.state.refresh_stop_event = stop_event
        app.state.refresh_thread = thread
        thread.start()
        # Shares the refresh loop's stop_event: the watchdog exists only to watch
        # that loop, so the two start and stop together.
        watchdog = threading.Thread(
            target=scheduler_watchdog_loop,
            args=(stop_event,),
            daemon=True,
        )
        app.state.scheduler_watchdog_thread = watchdog
        watchdog.start()
    else:
        app.state.refresh_stop_event = stop_event
        app.state.refresh_thread = None
        app.state.scheduler_watchdog_thread = None

    # Backfill durations for any existing YouTube entries not yet stored.
    def _backfill() -> None:
        with get_meta_connection() as conn:
            rows = conn.execute(
                "SELECT DISTINCT feed_url FROM folder_feeds WHERE feed_url LIKE '%youtube.com/feeds/videos.xml%'"
            ).fetchall()
        for row in rows:
            youtube_duration_service.fetch_and_store_durations_for_feed(str(row["feed_url"]))

    if not backfill_disabled:
        threading.Thread(
            target=lambda: _for_each_background_user("youtube-duration backfill", _backfill),
            daemon=True,
        ).start()

    # Backfill lead images for all feeds whose entries haven't been checked yet.
    # `force_retry_negative=False` keeps this incremental: only feeds that have
    # never been checked do work, so a startup right after an OPML import of
    # hundreds of feeds doesn't flood the meta DB with writes and starve the
    # request path.
    def _backfill_lead_images() -> None:
        with get_meta_connection() as conn:
            rows = conn.execute("SELECT DISTINCT feed_url FROM folder_feeds").fetchall()
        for row in rows:
            lead_image_service.fetch_and_store_lead_images_for_feed(str(row["feed_url"]), force_retry_negative=False)

    if not backfill_disabled:
        threading.Thread(
            target=lambda: _for_each_background_user("lead-image backfill", _backfill_lead_images),
            daemon=True,
        ).start()

    # Start the starred archive worker, then backfill pending rows for any
    # saved entries that don't yet have a complete archive (covers the
    # initial rollout and any re-stars after maintenance pruning). Also
    # one-shot metadata backfill for archive rows that completed before
    # title/link/etc columns were added to the schema.
    if not backfill_disabled:
        starred_archive_service.start_worker()

    def _archive_backfill_task() -> None:
        starred_archive_service.backfill_saved_entries_from_archive()
        starred_archive_service.backfill_missing_archives()
        starred_archive_service.backfill_metadata_for_complete_rows()

    if not backfill_disabled:
        threading.Thread(
            target=lambda: _for_each_background_user("starred-archive backfill", _archive_backfill_task),
            daemon=True,
            name="starred-archive-backfill",
        ).start()

    # One-time backfill: populate read_history from entry_read_state if history is empty.
    def _backfill_read_history() -> None:
        try:
            with get_meta_connection() as conn:
                count = conn.execute("SELECT COUNT(*) FROM read_history").fetchone()[0]
                if count > 0:
                    return
                rows = conn.execute(
                    "SELECT feed_url, entry_id, read_at FROM entry_read_state ORDER BY read_at DESC LIMIT ?",
                    (READ_HISTORY_CAP,),
                ).fetchall()
            if not rows:
                return
            needed = {(str(r["feed_url"]), str(r["entry_id"])): str(r["read_at"]) for r in rows}
            to_insert = []
            with get_reader() as reader:
                for (feed_url, entry_id), read_at in needed.items():
                    entry = reader.get_entry((feed_url, entry_id), None)
                    if entry is None:
                        continue
                    feed = reader.get_feed(feed_url, None)
                    to_insert.append(
                        (
                            feed_url,
                            entry_id,
                            str(getattr(entry, "title", None) or ""),
                            str(getattr(entry, "link", None) or ""),
                            str(getattr(feed, "resolved_title", None) or getattr(feed, "title", None) or ""),
                            read_at,
                        )
                    )
            if not to_insert:
                return
            with get_meta_connection() as conn:
                conn.executemany(
                    "INSERT INTO read_history (feed_url, entry_id, title, link, feed_title, read_at)"
                    " VALUES (?, ?, ?, ?, ?, ?)"
                    " ON CONFLICT(feed_url, entry_id) DO NOTHING",
                    to_insert,
                )
            LOGGER.info("[read_history] backfilled %d entries from entry_read_state", len(to_insert))
        except Exception:
            LOGGER.exception("[read_history] backfill error")

    if not backfill_disabled:
        threading.Thread(
            target=lambda: _for_each_background_user("read-history backfill", _backfill_read_history),
            daemon=True,
            name="read-history-backfill",
        ).start()

    # Daily Maintenance loop — runs once per day at the configured maintenance hour.
    # Gated by the same kill switch as the other daemons: the loop is a persistent
    # thread that ticks every 30s doing per-user DB writes (email-batch flush, and
    # a full maintenance run when due). Under the test suite (which sets
    # LECTIO_DISABLE_STARTUP_BACKFILL=1) it would otherwise keep running for the
    # rest of the pytest process once any TestClient test triggered the lifespan,
    # racing later tests' fresh-DB setup as an intermittent "database is locked".
    # Tests that need maintenance behavior call _run_daily_maintenance() directly.
    maint_stop_event = threading.Event()
    app.state.maint_stop_event = maint_stop_event
    if not backfill_disabled:
        threading.Thread(
            target=_daily_maintenance_loop,
            args=(maint_stop_event,),
            daemon=True,
            name="daily-maintenance",
        ).start()

    # In debug mode, auto-subscribe dev feeds to the _Lectio folder.
    if DEBUG_MODE:
        _dev_base = "http://127.0.0.1:8000"
        _dev_urls = [
            f"{_dev_base}/dev/feeds/email-match.xml",
            f"{_dev_base}/dev/feeds/email-match.atom",
            f"{_dev_base}/dev/feeds/email-match.json",
            f"{_dev_base}/dev/feeds/email-skip.xml",
            f"{_dev_base}/dev/feeds/email-skip.atom",
            f"{_dev_base}/dev/feeds/email-skip.json",
        ]
        with get_meta_connection() as conn:
            _lectio_folder_id = _get_lectio_folder_id(conn)
        if _lectio_folder_id:
            _dev_newly_added: list[str] = []
            for _dev_url in _dev_urls:
                try:
                    with get_meta_connection() as _c:
                        _already = _c.execute(
                            "SELECT 1 FROM folder_feeds WHERE folder_id = ? AND feed_url = ?",
                            (_lectio_folder_id, _dev_url),
                        ).fetchone()
                    if not _already:
                        add_feed_to_folder(_dev_url, _lectio_folder_id)
                        _dev_newly_added.append(_dev_url)
                except Exception:
                    LOGGER.exception("[dev-feeds] failed to subscribe %s", _dev_url)
            if _dev_newly_added:
                feed_refresh_service.update_feeds(_dev_newly_added)

    try:
        yield
    finally:
        stop_event = getattr(app.state, "refresh_stop_event", None)
        thread = getattr(app.state, "refresh_thread", None)
        if stop_event and thread:
            stop_event.set()
            # A scheduled refresh in flight can take tens of seconds (one HTTP
            # timeout per feed). Wait long enough to let it finish cleanly so
            # we don't leave the meta DB in a partial-write state.
            shutdown_timeout = float(os.getenv("LECTIO_SHUTDOWN_TIMEOUT_SECONDS", "30"))
            thread.join(timeout=shutdown_timeout)
            if thread.is_alive():
                LOGGER.warning(
                    "[shutdown] refresh worker did not finish within %.0fs; abandoning (daemon thread will be killed by interpreter exit)",
                    shutdown_timeout,
                )
        # Stop the starred archive worker — short timeout, since the only
        # work in flight is HTTP fetches with 15s ceilings.
        try:
            starred_archive_service.stop_worker(timeout=5.0)
        except Exception as exc:  # noqa: BLE001
            LOGGER.warning("[shutdown] starred archive worker stop failed: %s", exc)
        yt_stop = getattr(app.state, "yt_stop_event", None)
        if yt_stop:
            yt_stop.set()


app = FastAPI(title="Lectio", lifespan=lifespan)


def _session_logged_in(request: Request) -> bool:
    """Whether the session is fully authenticated (requires a valid user_id)."""
    if not request.session.get("authenticated"):
        return False
    uid = request.session.get("user_id")
    return bool(uid and tenancy.is_valid_user_id(uid))


class _AuthMiddleware(BaseHTTPMiddleware):
    async def dispatch(self, request: Request, call_next):
        if not AUTH_ENABLED:
            return await call_next(request)
        path = request.url.path
        if any(path.startswith(p) for p in _AUTH_EXEMPT_PREFIXES):
            return await call_next(request)
        if _session_logged_in(request):
            return await call_next(request)
        if AUTO_LOGIN:
            # Auto-authenticate as the first non-disabled admin user without
            # showing the login form. Intended for local/private-network installs.
            for _u in user_store.list_users():
                if _u.get("is_admin") and not _u.get("disabled"):
                    request.session["authenticated"] = True
                    request.session["user_id"] = _u["user_id"]
                    break
            if _session_logged_in(request):
                return await call_next(request)
        # Store a same-origin relative path (path + query) so _safe_next accepts
        # it and the user lands back on the deep link after logging in — e.g. a
        # /?subscribe=<feed> quick-subscription link. An absolute URL would be
        # rejected by _safe_next as an open-redirect and drop the user on "/".
        next_url = request.url.path
        if request.url.query:
            next_url += "?" + request.url.query
        return RedirectResponse(url=f"/login?next={quote_plus(next_url)}", status_code=303)


class _SecurityHeadersMiddleware:
    """Emit baseline security response headers from the app (opt-in via
    LECTIO_SECURITY_HEADERS=1), so they hold regardless of the reverse proxy.

    Mirrors the headers the bundled Traefik config sets. Existing headers are not
    overwritten, so a proxy that already sets them wins and there are no
    duplicates. HSTS is only added when LECTIO_HTTPS_ONLY=1."""

    def __init__(self, app, *, hsts: bool):
        self.app = app
        self._headers: list[tuple[bytes, bytes]] = [
            (b"x-content-type-options", b"nosniff"),
            (b"x-frame-options", b"DENY"),
            (b"referrer-policy", b"no-referrer-when-downgrade"),
        ]
        if hsts:
            self._headers.append((b"strict-transport-security", b"max-age=31536000; includeSubDomains"))

    async def __call__(self, scope, receive, send):
        if scope.get("type") != "http":
            return await self.app(scope, receive, send)

        async def send_wrapper(message):
            if message["type"] == "http.response.start":
                headers = message.setdefault("headers", [])
                present = {k.lower() for k, _ in headers}
                for key, val in self._headers:
                    if key not in present:
                        headers.append((key, val))
            await send(message)

        await self.app(scope, receive, send_wrapper)


def _touch_user_last_seen(uid: str) -> None:
    """Record per-user activity time, throttled (this runs on every request)."""
    now = time.time()
    with _last_seen_touch_lock:
        if now - _last_seen_touch.get(uid, 0.0) < _LAST_SEEN_THROTTLE_SECONDS:
            return
        _last_seen_touch[uid] = now
    try:
        user_store.touch_last_seen(uid, now)
    except Exception:
        LOGGER.debug("touch_last_seen failed", exc_info=True)


class _TenancyMiddleware:
    """Bind the request's user into the tenancy context.

    Pure-ASGI and registered innermost (downstream of the auth gate), so the
    binding wraps the route handler. Sync handlers run via anyio's threadpool,
    which copies the current contextvars into the worker thread, so a value set
    here is visible to get_reader() / get_meta_connection() deep in the call
    stack.

    Requests without an authenticated session user or API token (static assets,
    the login page, other unauthenticated hits) run unbound: anything they touch
    must be global, and a per-user DB access raises TenancyUnboundError.
    """

    def __init__(self, app):
        self.app = app

    @staticmethod
    def _greader_user_from_scope(scope) -> str | None:
        """Resolve a /greader/ request's bearer token to a user, using only the
        Authorization header / query string (no body read needed)."""
        token = ""
        for key, val in scope.get("headers", []):
            if key == b"authorization":
                auth = val.decode("latin-1")
                if auth.startswith("GoogleLogin auth="):
                    token = auth[17:].strip()
                elif auth.startswith("Bearer "):
                    token = auth[7:].strip()
                break
        if not token:
            qs = parse_qs(scope.get("query_string", b"").decode("latin-1"))
            token = (qs.get("token") or [""])[0]
        return user_store.resolve_greader_token(token)

    async def __call__(self, scope, receive, send):
        if scope.get("type") != "http":
            return await self.app(scope, receive, send)
        path = scope.get("path", "")
        uid: str | None = None
        if path.startswith("/greader/"):
            # API requests carry their identity in a bearer token, not a session.
            uid = self._greader_user_from_scope(scope)
        else:
            session = scope.get("session") or {}
            sid = session.get("user_id")
            if sid and session.get("authenticated") and tenancy.is_valid_user_id(sid):
                uid = sid
        if uid and tenancy.is_valid_user_id(uid):
            _touch_user_last_seen(uid)
            token = tenancy.set_current_user(uid)
            try:
                return await self.app(scope, receive, send)
            finally:
                tenancy.reset_current_user(token)
        return await self.app(scope, receive, send)


# Paths exempt from CSRF validation. /login is the auth gate itself (rate-
# limited separately). /static and /healthz are GET-only anyway, but listing
# explicitly documents intent.
_CSRF_EXEMPT_PREFIXES = (
    "/login",
    "/static",
    "/healthz",
    "/api/img",
    "/api/favicon",
    "/api/save",
    "/api/bookmarklet/save",
    "/fever",
    "/greader/",
    "/v1/",
    "/websub/",
)
_CSRF_SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
_CSRF_SESSION_KEY = "csrf_token"
_CSRF_HEADER_NAME = "x-csrf-token"
_CSRF_FORM_FIELD = "_csrf"


def _ensure_csrf_token(session: dict) -> str:
    """Return the session's CSRF token, generating one on first use.

    Token is per-session (rotated whenever the session itself rotates, e.g.
    on logout/login). Stored in the session cookie which is HttpOnly + signed
    so attackers on other origins can neither read nor forge it.
    """
    token = session.get(_CSRF_SESSION_KEY)
    if not token:
        token = secrets.token_urlsafe(32)
        session[_CSRF_SESSION_KEY] = token
    return token


class _AccessLogMiddleware:
    """Replaces uvicorn's built-in access log so we can suppress the prefetch
    204/503s without wrestling with uvicorn's logger config (filters added to
    uvicorn.access weren't taking effect in this deployment).

    Format mirrors uvicorn's: ``CLIENT - "METHOD PATH HTTP/x.y" STATUS REASON``.
    """

    _SUPPRESS_STATUSES = (204, 503)
    _STATUS_REASONS = {
        200: "OK",
        204: "No Content",
        301: "Moved Permanently",
        302: "Found",
        303: "See Other",
        304: "Not Modified",
        307: "Temporary Redirect",
        308: "Permanent Redirect",
        400: "Bad Request",
        401: "Unauthorized",
        403: "Forbidden",
        404: "Not Found",
        405: "Method Not Allowed",
        409: "Conflict",
        413: "Payload Too Large",
        429: "Too Many Requests",
        500: "Internal Server Error",
        502: "Bad Gateway",
        503: "Service Unavailable",
    }

    def __init__(self, app):
        self.app = app

    async def __call__(self, scope, receive, send):
        if scope["type"] != "http":
            await self.app(scope, receive, send)
            return

        _t0 = time.monotonic()
        status_holder = [0]

        async def send_wrapper(message):
            if message.get("type") == "http.response.start":
                status_holder[0] = int(message.get("status", 0))
            await send(message)

        try:
            await self.app(scope, receive, send_wrapper)
        finally:
            status = status_holder[0]
            elapsed_ms = int((time.monotonic() - _t0) * 1000)
            client = scope.get("client") or ("?", 0)
            client_addr = f"{client[0]}:{client[1]}"
            method = scope.get("method", "?")
            raw_path = scope.get("raw_path", b"") or scope.get("path", "").encode()
            qs = scope.get("query_string", b"") or b""
            request_path = raw_path.decode("latin-1", errors="replace")
            if qs:
                request_path = f"{request_path}?{qs.decode('latin-1', errors='replace')}"
            http_version = scope.get("http_version", "1.1")
            request_line = f"{method} {request_path} HTTP/{http_version}"
            # Suppression: prefetch 204/503 with list_feed_url= are pure noise.
            if not (status in self._SUPPRESS_STATUSES and "list_feed_url=" in request_path):
                reason = self._STATUS_REASONS.get(status, "")
                duration = f" {elapsed_ms}ms" if elapsed_ms >= 200 else ""
                LOGGER.info('%s - "%s" %d %s%s', client_addr, request_line, status, reason, duration)


class _RejectPrefetchMiddleware:
    """Two-layer defense against browser link-prefetch floods.

    1. **Drop prefetches outright.** Browsers tag them with one of several
       headers (Sec-Purpose, Purpose, X-Moz, Sec-Fetch-Dest=empty for nav-style
       prefetch). Match generously — false positives just make a prefetch miss,
       which is fine.
    2. **Concurrency cap on home() requests.** Backstop in case detection
       misses a browser variant: hold a semaphore so only N home renders run
       at once. Excess returns 503, which tells the prefetcher to stop. /healthz
       and /static bypass both layers so probes and asset prefetches still work.
    """

    PASSTHROUGH_PREFIXES = ("/healthz", "/static", "/login", "/api/img", "/api/favicon")

    def __init__(self, app):
        self.app = app

    async def __call__(self, scope, receive, send):
        if scope["type"] != "http":
            await self.app(scope, receive, send)
            return
        path = scope.get("path", "")
        if any(path.startswith(p) for p in self.PASSTHROUGH_PREFIXES):
            await self.app(scope, receive, send)
            return

        headers = scope.get("headers") or []
        is_prefetch = False
        is_spa_fetch = False
        sec_fetch_dest = b""
        sec_fetch_mode = b""
        x_requested_with = b""
        user_agent = b""
        referer = b""
        for header_name, header_value in headers:
            name = header_name.decode("latin-1").lower()
            if name == "sec-purpose":
                if b"prefetch" in header_value.lower() or b"prerender" in header_value.lower():
                    is_prefetch = True
            elif name == "purpose":
                low = header_value.lower()
                if b"prefetch" in low or b"preview" in low or b"prerender" in low:
                    is_prefetch = True
            elif name == "x-moz":
                if b"prefetch" in header_value.lower() or b"prerender" in header_value.lower():
                    is_prefetch = True
            elif name == "sec-fetch-dest":
                sec_fetch_dest = header_value
            elif name == "sec-fetch-mode":
                sec_fetch_mode = header_value
            elif name == "x-requested-with":
                x_requested_with = header_value
                if b"lectio" in header_value.lower():
                    is_spa_fetch = True
            elif name == "user-agent":
                user_agent = header_value
            elif name == "referer":
                referer = header_value
        # Heuristic: browser-side prefetch via fetch() (Chrome NoState Prefetch
        # and similar) doesn't always set Sec-Purpose, but it has a unique
        # fingerprint — Sec-Fetch-Dest=empty + Sec-Fetch-Mode=cors on a path
        # that isn't an actual API endpoint, with no X-Requested-With from our
        # SPA. Real SPA fetches always send X-Requested-With=lectio-...; real
        # top-level navigations send Sec-Fetch-Dest=document Sec-Fetch-Mode=navigate.
        if not is_prefetch and not is_spa_fetch and sec_fetch_dest == b"empty" and sec_fetch_mode == b"cors" and path == "/":
            is_prefetch = True

        # Diagnostic: dump the first few suspect-looking requests so we can
        # see what's actually firing them. Capped to avoid log spam.
        with _prefetch_header_log_lock:
            remaining = _prefetch_header_log_remaining[0]
        if remaining > 0:
            with _prefetch_header_log_lock:
                if _prefetch_header_log_remaining[0] > 0:
                    _prefetch_header_log_remaining[0] -= 1
                    LOGGER.info(
                        "[prefetch-diag] path=%s qs=%r dest=%r mode=%r xrw=%r referer=%r ua=%r",
                        path,
                        scope.get("query_string", b"")[:200],
                        sec_fetch_dest,
                        sec_fetch_mode,
                        x_requested_with,
                        referer[:120],
                        user_agent[:120],
                    )

        if is_prefetch:
            response = Response(status_code=204, headers={"Cache-Control": "no-store"})
            await response(scope, receive, send)
            return

        if path == "/" or path.startswith("/?"):
            await self.app(scope, receive, send)
            return

        await self.app(scope, receive, send)


class _CSRFMiddleware:
    """Pure-ASGI CSRF protection.

    On unsafe methods (POST/PUT/PATCH/DELETE) requires a token that matches
    the session's stored token. Token is supplied either via:
      - X-CSRF-Token header (used by SPA fetch handlers), or
      - `_csrf` form field (for plain HTML form submits / multipart uploads).

    Read-only methods are allowed through unchanged. The body is buffered and
    re-streamed to downstream handlers via a replay receive callable so route
    handlers can still parse it normally.
    """

    def __init__(self, app):
        self.app = app

    async def __call__(self, scope, receive, send):
        if scope["type"] != "http":
            await self.app(scope, receive, send)
            return

        # Ensure every request has a CSRF token in its session so templates
        # can render it and SPA fetches can read it from a meta tag. The
        # token is created lazily on first request after a fresh session.
        session = scope.get("session")
        if session is not None and not session.get(_CSRF_SESSION_KEY):
            session[_CSRF_SESSION_KEY] = secrets.token_urlsafe(32)

        method = scope.get("method", "GET").upper()
        path = scope.get("path", "")

        if method in _CSRF_SAFE_METHODS or any(path.startswith(p) for p in _CSRF_EXEMPT_PREFIXES):
            await self.app(scope, receive, send)
            return

        # Buffer the request body so downstream handlers can still read it.
        body_chunks: list[bytes] = []
        more_body = True
        while more_body:
            message = await receive()
            if message["type"] != "http.request":
                # Unexpected (e.g. http.disconnect); pass through unchanged.
                await self.app(scope, receive, send)
                return
            body_chunks.append(message.get("body", b""))
            more_body = bool(message.get("more_body", False))
        body = b"".join(body_chunks)

        # Token from header (preferred for SPA fetches).
        submitted: str | None = None
        for header_name, header_value in scope.get("headers") or []:
            if header_name.decode("latin-1").lower() == _CSRF_HEADER_NAME:
                submitted = header_value.decode("latin-1")
                break

        # Fall back to a `_csrf` field in form-encoded bodies.
        if not submitted and body:
            content_type = b""
            for header_name, header_value in scope.get("headers") or []:
                if header_name.decode("latin-1").lower() == "content-type":
                    content_type = header_value
                    break
            ctype = content_type.decode("latin-1", errors="ignore").lower()
            if ctype.startswith("application/x-www-form-urlencoded"):
                from urllib.parse import parse_qs

                try:
                    form = parse_qs(body.decode("utf-8", errors="ignore"))
                    values = form.get(_CSRF_FORM_FIELD, [])
                    if values:
                        submitted = values[0]
                except Exception:
                    submitted = None
            elif ctype.startswith("multipart/form-data"):
                # Cheap multipart probe: scan the raw body for the field name.
                # Avoids pulling in a streaming parser for the one OPML form.
                marker = b'name="' + _CSRF_FORM_FIELD.encode() + b'"'
                idx = body.find(marker)
                if idx >= 0:
                    # The value sits two CRLFs after the disposition line.
                    sep = b"\r\n\r\n"
                    val_start = body.find(sep, idx)
                    if val_start >= 0:
                        val_start += len(sep)
                        val_end = body.find(b"\r\n", val_start)
                        if val_end >= 0:
                            try:
                                submitted = body[val_start:val_end].decode("utf-8", errors="ignore")
                            except Exception:
                                submitted = None

        expected = (scope.get("session") or {}).get(_CSRF_SESSION_KEY)
        if not expected or not submitted or not secrets.compare_digest(str(submitted), str(expected)):
            response = JSONResponse(
                {"detail": "CSRF token missing or invalid."},
                status_code=403,
            )
            await response(scope, receive, send)
            return

        # Replay the buffered body to downstream handlers.
        replayed = False

        async def replay_receive():
            nonlocal replayed
            if not replayed:
                replayed = True
                return {"type": "http.request", "body": body, "more_body": False}
            return {"type": "http.disconnect"}

        await self.app(scope, replay_receive, send)


# Middleware order matters. Starlette's add_middleware does insert(0) and the
# stack-builder reverses, so LAST add_middleware = OUTERMOST = runs FIRST
# inbound. We need this inbound chain:
#   _RejectPrefetchMiddleware (drop browser speculation-rules prefetches)
#   ProxyHeaders (rewrite client IP / scheme from X-Forwarded-*)
#   → SessionMiddleware (parse signed session cookie → scope["session"])
#   → _CSRFMiddleware (validate POST tokens; needs scope["session"])
#   → _AuthMiddleware (gate on session["authenticated"]; needs scope["session"])
#   → app
# Therefore add Auth FIRST (innermost) and outer middlewares LAST.
# _TenancyMiddleware is added before Auth so it is innermost of all — it binds
# the per-user context right around the endpoint, after the auth gate has run.
app.add_middleware(_TenancyMiddleware)
app.add_middleware(_AuthMiddleware)
app.add_middleware(_CSRFMiddleware)
app.add_middleware(
    SessionMiddleware,
    secret_key=SESSION_SECRET_KEY,
    max_age=SESSION_MAX_AGE_SECONDS,
    https_only=_HTTPS_ONLY,
    same_site="lax",
)
# When running behind Traefik/nginx/Caddy, trust forwarded headers so
# uvicorn sees the real scheme and client IP.
if _HTTPS_ONLY:
    from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware

    _trusted_hosts = "*" if _TRUSTED_PROXIES == "*" else [h.strip() for h in _TRUSTED_PROXIES.split(",") if h.strip()]
    app.add_middleware(ProxyHeadersMiddleware, trusted_hosts=_trusted_hosts)
# Drop prefetch traffic before any other middleware does work.
app.add_middleware(_RejectPrefetchMiddleware)
# App-emitted security headers (opt-in), outermost so they apply to every
# response regardless of which inner middleware produced it.
if _SECURITY_HEADERS_ENABLED:
    app.add_middleware(_SecurityHeadersMiddleware, hsts=_HTTPS_ONLY)
# Compress responses before they reach Traefik so Traefik's own compress
# middleware sees Content-Encoding: gzip and passes through without buffering.
# Buffering the full 500KB page before sending a single byte was the main
# culprit for slow first-byte times on variable connections.
app.add_middleware(GZipMiddleware, minimum_size=1000)
# Access log replaces uvicorn's built-in (disabled via --no-access-log) so we
# can suppress prefetch 204/503 noise. Add LAST so it sees the final response.
app.add_middleware(_AccessLogMiddleware)


class _CachedStaticFiles(StaticFiles):
    """StaticFiles that sends a long-lived Cache-Control header.

    Safe because every <link>/<script> URL includes a `?v={STATIC_ASSET_VERSION}`
    cache-buster — bump the version to invalidate. Vendored third-party assets
    under static/vendor/<lib>-<version>/ (e.g. KaTeX) are the exception: they
    carry no `?v=` query param at all, since a vendored release is immutable —
    a version bump there means a new directory name, not a hash recompute, so
    the long-lived cache header is still safe without the query-string scheme.
    """

    async def get_response(self, path: str, scope):
        response = await super().get_response(path, scope)
        if response.status_code == 200:
            response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
        return response


# Python's mimetypes table predates .webmanifest, so StaticFiles would serve the
# PWA manifest as octet-stream and Chrome would refuse to install the app.
mimetypes.add_type("application/manifest+json", ".webmanifest")

app.mount("/static", _CachedStaticFiles(directory=str(BASE_DIR / "static")), name="static")
templates = Jinja2Templates(directory=str(BASE_DIR / "templates"))
templates.env.filters["urlencode"] = lambda value: quote_plus(str(value))


def _csrf_token_for(request: Request) -> str:
    """Read the request's CSRF token; ensure one exists if the session lacks it."""
    return _ensure_csrf_token(request.session)


def _csrf_input(request: Request) -> str:
    """Render a hidden CSRF input for inclusion in <form> blocks."""
    token = _csrf_token_for(request)
    return f'<input type="hidden" name="{_CSRF_FORM_FIELD}" value="{token}" />'


# Expose helpers as Jinja globals so templates can call csrf_input(request).
# ty's strict view of globals' value-type doesn't accept arbitrary callables;
# Jinja itself does. Suppress the false positives.
templates.env.globals["csrf_input"] = _csrf_input  # ty: ignore[invalid-assignment]
templates.env.globals["csrf_token_for"] = _csrf_token_for  # ty: ignore[invalid-assignment]


@dataclass
class FolderOption:
    id: int
    name: str
    path: str
    depth: int


@dataclass
class FeedInFolder:
    url: str
    title: str
    icon_url: str | None
    unread_count: int
    has_error: bool = False
    disabled: bool = False


_meta_conn_local = threading.local()

# Refresh-contention investigation (Plan.md Tier 1) -- see
# services.reader_api's identical _TimedConnection for the full rationale.
# Kept as a separate copy rather than importing reader_api's: it wall-clock-
# times every statement issued over the META connection specifically, so a
# slow meta_block gets its own "db=meta" tag distinguishing it from the
# reader-DB one.
_SLOW_SQL_MS = 250

# "Read Above/Below" lead: elapsed_ms alone can't distinguish a genuinely slow
# query from one that spent nearly all its wall time in SQLite's busy-handler
# retry loop -- both look identical from outside. The progress handler fires
# every _PROGRESS_HANDLER_N VM instructions *during execution*, not during the
# busy-wait, so near-zero steps on a "slow" query is direct proof of lock-wait.
# Read-only: the handler always returns 0, never aborting a statement.
_PROGRESS_HANDLER_N = 1000


class _TimedMetaCursor(sqlite3.Cursor):
    def execute(self, sql, parameters=()):
        conn = cast("_TimedMetaConnection", self.connection)
        conn._lectio_progress_steps = 0
        self._lectio_sql = sql
        start = time.perf_counter()
        try:
            return super().execute(sql, parameters)
        finally:
            elapsed_ms = (time.perf_counter() - start) * 1000
            if elapsed_ms > _SLOW_SQL_MS:
                LOGGER.info(
                    "[perf] slow_sql db=meta elapsed_ms=%d progress_steps=%d sql=%s",
                    int(elapsed_ms),
                    conn._lectio_progress_steps,
                    " ".join(str(sql).split())[:200],
                )

    def executemany(self, sql, parameters):
        conn = cast("_TimedMetaConnection", self.connection)
        conn._lectio_progress_steps = 0
        start = time.perf_counter()
        try:
            return super().executemany(sql, parameters)
        finally:
            elapsed_ms = (time.perf_counter() - start) * 1000
            if elapsed_ms > _SLOW_SQL_MS:
                LOGGER.info(
                    "[perf] slow_sql db=meta elapsed_ms=%d progress_steps=%d executemany sql=%s",
                    int(elapsed_ms),
                    conn._lectio_progress_steps,
                    " ".join(str(sql).split())[:200],
                )

    def fetchall(self):
        # `execute()` above only times statement *preparation*; a plain SELECT
        # doesn't eagerly scan, so the real cost of `conn.execute(...).fetchall()`
        # (the dominant read pattern in this codebase) happened here, previously
        # invisible to slow-SQL logging entirely -- found 2026-09-03 chasing a
        # live stall that traced back to exactly this shape (Plan.md Tier 1).
        # Separate before/after progress-step snapshot so this doesn't double
        # count whatever execute() already attributed to itself.
        conn = cast("_TimedMetaConnection", self.connection)
        steps_before = conn._lectio_progress_steps
        start = time.perf_counter()
        try:
            return super().fetchall()
        finally:
            elapsed_ms = (time.perf_counter() - start) * 1000
            if elapsed_ms > _SLOW_SQL_MS:
                sql = getattr(self, "_lectio_sql", "?")
                LOGGER.info(
                    "[perf] slow_sql db=meta elapsed_ms=%d progress_steps=%d fetchall sql=%s",
                    int(elapsed_ms),
                    conn._lectio_progress_steps - steps_before,
                    " ".join(str(sql).split())[:200],
                )


class _TimedMetaConnection(sqlite3.Connection):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._lectio_progress_steps = 0
        self.set_progress_handler(self._lectio_on_progress, _PROGRESS_HANDLER_N)

    def _lectio_on_progress(self) -> int:
        self._lectio_progress_steps += 1
        return 0  # never abort the statement

    def cursor(self, factory=None):
        return super().cursor(factory or _TimedMetaCursor)

    def execute(self, sql, parameters=()):
        # Delegate to self.cursor() rather than the sqlite3.Connection.execute()
        # shortcut directly: the shortcut returns a bare sqlite3.Cursor (verified
        # -- it does not route through the overridden cursor() factory at all),
        # so `conn.execute(...).fetchall()` -- the dominant read pattern in this
        # module -- was silently bypassing _TimedMetaCursor's fetchall() override
        # above. Going through self.cursor().execute(...) guarantees callers
        # always get a _TimedMetaCursor back.
        return self.cursor().execute(sql, parameters)

    def executemany(self, sql, parameters):
        # Batched-write flush points (lead images, alt/title, entry_read_state)
        # use conn.executemany directly -- routed through self.cursor() for the
        # same reason execute() is above.
        return self.cursor().executemany(sql, parameters)


def get_meta_connection() -> sqlite3.Connection:
    """Per-(thread, user) persistent SQLite connection to the meta DB.

    The target DB is resolved through the tenancy seam from the current user
    (defaults to the single legacy user). Each worker thread keeps one
    connection per user_id it has served.

    Opening a fresh connection per request was a measured bottleneck: simple
    SELECTs that take 5ms standalone took 2-3s under 8-way concurrency because
    every request paid for file open, schema load, mmap setup, and PRAGMA
    application. Each uvicorn worker thread now opens once and reuses, so the
    cost is paid 8-40 times total instead of N-times-per-request.

    Note: sqlite3.Connection's context-manager protocol commits the transaction
    on success but does NOT close the connection — the existing
    `with get_meta_connection() as conn:` pattern continues to work and is
    transaction-scoped per call site, while the underlying connection persists.
    """
    uid = tenancy.current_user_id()
    pool = getattr(_meta_conn_local, "pool", None)
    if pool is None:
        pool = {}
        _meta_conn_local.pool = pool
    conn = pool.get(uid)
    if conn is not None:
        return conn
    _meta_factory = _TimedMetaConnection if LECTIO_PERF_DEBUG else sqlite3.Connection
    conn = sqlite3.connect(str(tenancy.meta_db_path(uid)), timeout=10.0, factory=_meta_factory)
    conn.row_factory = sqlite3.Row
    # WAL + busy_timeout so overlapping writers (e.g. background refresh writing
    # folder_feeds while a request persists a setting) wait briefly instead of
    # immediately failing with "database is locked".
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=10000")
    # Keep WAL small: checkpoint at 200 pages (~800KB) rather than the default
    # 1000 pages so the file never balloons to tens of MB between restarts.
    # Raised to 1000 and reverted same day, 2026-09-03 (Plan.md Tier 1, Read
    # Above/Below lead) -- see services/reader_api.py's _LectioReaderStorage
    # docstring for the full writeup: the experiment showed no benefit (a live
    # capture right after deploy was worse, not better) and a follow-up
    # capture found near-zero real work across many different queries in both
    # DBs, arguing against checkpoint frequency as the mechanism.
    conn.execute("PRAGMA wal_autocheckpoint=200")
    # Whenever a checkpoint restarts the WAL, cut the file back to this size instead of leaving a vacuum-sized one behind.
    conn.execute("PRAGMA journal_size_limit=67108864")
    # synchronous=NORMAL 2026-09-03 (Plan.md Tier 1, Read Above/Below lead),
    # replacing SQLite's compiled-in FULL default -- see
    # services/reader_api.py's _LectioReaderStorage docstring for the full
    # writeup; kept in sync here since both connections share the theory.
    conn.execute("PRAGMA synchronous=NORMAL")
    pool[uid] = conn
    return conn


_websub_conn_local = threading.local()
WEBSUB_DB_PATH = DATA_DIR / "lectio_websub.sqlite"


def get_websub_connection() -> sqlite3.Connection:
    """Per-thread connection to the shared (non-per-user) WebSub subscription DB."""
    pool = getattr(_websub_conn_local, "pool", None)
    if pool is not None:
        return pool
    conn = sqlite3.connect(str(WEBSUB_DB_PATH), timeout=10.0)
    conn.row_factory = sqlite3.Row
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=10000")
    conn.execute("PRAGMA wal_autocheckpoint=200")
    _websub_conn_local.pool = conn
    return conn


def get_thumb_connection() -> sqlite3.Connection:
    conn = sqlite3.connect(str(THUMB_DB_PATH))
    conn.row_factory = sqlite3.Row
    return conn


def ensure_websub_schema() -> None:
    """Create the shared (non-per-user) WebSub tables if they don't exist."""
    with get_websub_connection() as conn:
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS websub_subscriptions (
                feed_url      TEXT PRIMARY KEY,
                hub_url       TEXT,
                secret        TEXT,
                subscribed_at REAL,
                verified      INTEGER DEFAULT 0,
                expires_at    REAL,
                lease_seconds INTEGER,
                hub_tried_at  REAL
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS websub_subscribers (
                feed_url TEXT NOT NULL,
                user_id  TEXT NOT NULL,
                PRIMARY KEY (feed_url, user_id)
            )
            """
        )


def _migrate_websub_to_shared() -> None:
    """One-time migration: copy per-user websub_subscriptions rows to the shared DB.

    Idempotent (INSERT OR IGNORE) so safe to re-run on every startup.
    After this runs, the service only writes to the shared DB."""
    for uid in _background_user_ids():
        with tenancy.user_context(uid):
            try:
                conn = get_meta_connection()
                rows = conn.execute(
                    "SELECT feed_url, hub_url, secret, subscribed_at, verified, "
                    "expires_at, lease_seconds, hub_tried_at "
                    "FROM websub_subscriptions"
                ).fetchall()
            except Exception:
                continue  # table may not exist for new users
            if not rows:
                continue
            with get_websub_connection() as wconn:
                for r in rows:
                    wconn.execute(
                        "INSERT OR IGNORE INTO websub_subscriptions "
                        "(feed_url, hub_url, secret, subscribed_at, verified, "
                        "expires_at, lease_seconds, hub_tried_at) "
                        "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
                        (
                            r["feed_url"],
                            r["hub_url"],
                            r["secret"],
                            r["subscribed_at"],
                            r["verified"],
                            r["expires_at"],
                            r["lease_seconds"],
                            r["hub_tried_at"],
                        ),
                    )
                    wconn.execute(
                        "INSERT OR IGNORE INTO websub_subscribers (feed_url, user_id) VALUES (?, ?)",
                        (r["feed_url"], uid),
                    )


def ensure_thumb_schema() -> None:
    with get_thumb_connection() as conn:
        # WAL allows concurrent readers; thumb content is regeneratable so
        # synchronous=NORMAL is the right tradeoff (durability not critical).
        conn.execute("PRAGMA journal_mode=WAL")
        conn.execute("PRAGMA synchronous=NORMAL")
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS thumb_cache (
                cache_key TEXT PRIMARY KEY,
                jpeg BLOB NOT NULL,
                created_at REAL NOT NULL
            )
            """
        )


def get_img_cache_connection() -> sqlite3.Connection:
    """Connection to the global (shared) /api/img proxy cache."""
    conn = sqlite3.connect(str(IMG_CACHE_DB_PATH))
    conn.row_factory = sqlite3.Row
    return conn


def ensure_img_cache_schema() -> None:
    with get_img_cache_connection() as conn:
        # WAL for concurrent readers; cached bytes are regeneratable so
        # synchronous=NORMAL is the right durability/throughput tradeoff.
        conn.execute("PRAGMA journal_mode=WAL")
        conn.execute("PRAGMA synchronous=NORMAL")
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS img_cache (
                cache_key TEXT PRIMARY KEY,
                content_type TEXT NOT NULL,
                body BLOB NOT NULL,
                size INTEGER NOT NULL,
                created_at REAL NOT NULL,
                last_accessed REAL NOT NULL
            )
            """
        )
        # Eviction scans by last_accessed; index keeps the daily sweep cheap.
        conn.execute("CREATE INDEX IF NOT EXISTS idx_img_cache_last_accessed ON img_cache(last_accessed)")


def get_yt_duration_connection() -> sqlite3.Connection:
    """Connection to the global (shared) YouTube-duration cache."""
    conn = sqlite3.connect(str(YT_DURATION_DB_PATH))
    conn.row_factory = sqlite3.Row
    return conn


def ensure_yt_duration_schema() -> None:
    with get_yt_duration_connection() as conn:
        conn.execute("PRAGMA journal_mode=WAL")
        conn.execute("PRAGMA synchronous=NORMAL")
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS youtube_video_duration (
                video_id TEXT PRIMARY KEY,
                duration_seconds INTEGER,
                duration_display TEXT,
                fetched_at TEXT NOT NULL DEFAULT (datetime('now'))
            )
            """
        )
        # Premiere/live status, fetched alongside duration from the same
        # videos.list call (snippet.liveBroadcastContent, liveStreamingDetails.
        # scheduledStartTime) — used to detect videos that haven't aired yet.
        try:
            conn.execute("ALTER TABLE youtube_video_duration ADD COLUMN live_broadcast_content TEXT")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE youtube_video_duration ADD COLUMN scheduled_start_time TEXT")
        except Exception:
            pass
        # Members-only status: NOT in the videos.list response the other columns
        # ride along on for free — the Data API has no field for it, and the
        # "members" endpoint only works for channels you own. Detected instead
        # by fetching the video's own watch page and checking for its
        # BADGE_STYLE_TYPE_MEMBERS_ONLY badge. NULL = never checked, so a check
        # only costs a fetch once per video, ever (see fetch_and_cache_members_only).
        try:
            conn.execute("ALTER TABLE youtube_video_duration ADD COLUMN members_only INTEGER")
        except Exception:
            pass


def get_starred_archive_connection() -> sqlite3.Connection:
    # Per-user DB (resolved via tenancy); returns a fresh connection per call.
    conn = sqlite3.connect(str(tenancy.starred_archive_db_path()))
    conn.row_factory = sqlite3.Row
    return conn


@contextmanager
def archive_conn() -> Generator[sqlite3.Connection]:
    """An archive connection that commits (or rolls back) *and* closes.

    ``with sqlite3_connection:`` only wraps a transaction — it leaves the
    handle open, and this factory returns a fresh one per call. Use this
    wherever the archive DB is touched.
    """
    conn = get_starred_archive_connection()
    try:
        with conn:
            yield conn
    finally:
        conn.close()


def ensure_starred_archive_schema() -> None:
    with archive_conn() as conn:
        # Archive content is irreplaceable if the source goes down — leave
        # synchronous at the default FULL for durability. WAL still helps with
        # concurrent reads from the render path.
        conn.execute("PRAGMA journal_mode=WAL")
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS archived_entry (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                status TEXT NOT NULL,
                starred_at REAL NOT NULL,
                archived_at REAL,
                error TEXT,
                source_html_zlib BLOB,
                readability_html_zlib BLOB,
                content_html_zlib BLOB,
                title TEXT,
                link TEXT,
                feed_title TEXT,
                author TEXT,
                published_at REAL,
                received_at REAL,
                PRIMARY KEY (feed_url, entry_id)
            )
            """
        )
        # Migration: add columns to pre-existing tables created before the
        # orphan-save support landed. PRAGMA table_info is the standard way
        # to detect what's already there.
        existing_cols = {row["name"] for row in conn.execute("PRAGMA table_info(archived_entry)").fetchall()}
        for col_name, col_decl in (
            ("content_html_zlib", "BLOB"),
            ("title", "TEXT"),
            ("link", "TEXT"),
            ("feed_title", "TEXT"),
            ("author", "TEXT"),
            ("published_at", "REAL"),
            ("received_at", "REAL"),
            # Maintained, not computed live -- written once at archive-completion
            # time (which capture AND re-fetch both funnel through), not derived
            # from LENGTH()+join on every render. NULL until the first successful
            # archive. See Plan.md "Saved: see and sort by item size".
            ("content_size_bytes", "INTEGER"),
            # How the source-page fetch went at capture time: ok, no_link, http_<code>, timeout, tls, connect, blocked,
            # error. NULL for rows captured before this existed. A failed fetch still archives as 'complete' (the feed's
            # own content may be all there is), so this is the only record of WHY a complete archive can be empty.
            ("source_fetch_status", "TEXT"),
            # Nightly retry bookkeeping for transient fetch failures (see StarredArchiveService.requeue_transient_failures).
            ("capture_retries", "INTEGER NOT NULL DEFAULT 0"),
            ("capture_retried_at", "REAL"),
            ("capture_retry_pending", "INTEGER NOT NULL DEFAULT 0"),
        ):
            if col_name not in existing_cols:
                conn.execute(f"ALTER TABLE archived_entry ADD COLUMN {col_name} {col_decl}")
        conn.execute("CREATE INDEX IF NOT EXISTS idx_archived_entry_status ON archived_entry (status)")
        # Covers get_capture_failed_keys, so it never reads the capture blobs.
        conn.execute(
            "CREATE INDEX IF NOT EXISTS idx_archived_entry_capture"
            " ON archived_entry (source_fetch_status, status, content_size_bytes, feed_url, entry_id)"
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS archived_asset (
                asset_hash TEXT PRIMARY KEY,
                data BLOB NOT NULL,
                content_type TEXT NOT NULL,
                width INTEGER,
                height INTEGER,
                byte_size INTEGER NOT NULL,
                created_at REAL NOT NULL
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS archived_asset_link (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                source_url TEXT NOT NULL,
                asset_hash TEXT NOT NULL,
                PRIMARY KEY (feed_url, entry_id, source_url)
            )
            """
        )
        conn.execute("CREATE INDEX IF NOT EXISTS idx_archived_asset_link_hash ON archived_asset_link (asset_hash)")


def ensure_reader_indexes() -> None:
    """Add Lectio-owned performance indexes to the current tenant's reader DB.

    reader manages the `entries` table; we add a partial index it doesn't ship.
    The per-feed unread-count query (`WHERE read=0 GROUP BY feed`) otherwise scans
    the whole entries table (75k+ rows at scale) on each cache refresh, which under
    refresh write-contention stretched to several seconds and made the UI sluggish.
    A partial covering index over only the unread rows makes it a ~1k-row scan.
    Idempotent and safe (an extra index reader never drops)."""
    try:
        conn = sqlite3.connect(str(tenancy.reader_db_path()), timeout=5)
        try:
            has_entries = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='entries'").fetchone()
            if has_entries:
                conn.execute("CREATE INDEX IF NOT EXISTS entries_unread_by_feed ON entries(feed) WHERE read=0")
                conn.commit()
        finally:
            conn.close()
    except Exception:
        LOGGER.debug("ensure_reader_indexes failed", exc_info=True)


def provision_user_storage(user_id: str) -> None:
    """Create and schema-init a user's isolated databases (keyed by stable
    user_id).

    Routes through the tenancy seam: under ``user_context`` the schema-init
    helpers and reader open resolve to this user's own DB files. Idempotent —
    safe to call on an already-provisioned user.
    """
    tenancy.ensure_user_data_dir(user_id)
    with tenancy.user_context(user_id):
        ensure_meta_schema()
        ensure_starred_archive_schema()
        # reader builds its own schema on first client(); opening provisions it.
        with get_reader():
            pass
        ensure_reader_indexes()


def delete_user_storage(user_id: str) -> None:
    """Recursively remove a user's isolated data directory
    (``DATA_DIR/users/<user_id>/`` and all its DBs).

    Content-addressed caches (thumbnails, image proxy, lead-image/strategy
    results) are global and hold no per-user data, so they are intentionally
    left untouched.
    """
    data_dir = tenancy.user_data_dir(user_id)
    if data_dir.exists():
        shutil.rmtree(data_dir, ignore_errors=True)


def _seed_admin_integrations_from_env(admin_id: str) -> None:
    """Copy env-provided per-user integration defaults into the admin's settings,
    so a fresh multi-user instance has the admin configured (env is "used for
    bootstrap then ignored" for these per-user values). Only seeds keys not
    already set; the instance-shared Resend API key is NOT seeded here (it stays
    env-read for everyone via get_resend_api_key)."""
    seeds = [
        (SETTING_EMAIL_FROM, _ENV_RESEND_FROM),
        (EMAIL_TO_SETTING_KEY, _ENV_RESEND_TO),
        # YouTube is per-user; the env vars seed the first (admin) user only.
        (SETTING_YT_API_KEY, _ENV_YT_API_KEY),
        (SETTING_YT_CHANNEL_ID, _ENV_YT_CHANNEL_ID),
        (SETTING_YT_FOLDER_NAME, _ENV_YT_FOLDER_NAME),
    ]
    with tenancy.user_context(admin_id):
        with get_meta_connection() as conn:
            for key, val in seeds:
                if val and not get_setting(conn, key):
                    set_setting(conn, key, val)


def bootstrap_admin() -> None:
    """Seed the bootstrap admin on first startup when no users exist.

    No-op once any user exists. Provisions the new admin's isolated storage
    and warns loudly if the default password is still in use.
    """
    if user_store.count() > 0:
        return
    username = BOOTSTRAP_ADMIN_USERNAME
    if not tenancy.is_valid_user_id(username):
        LOGGER.error(
            "Cannot bootstrap admin: LECTIO_ADMIN_USERNAME=%r is not a valid username (must match {A-Za-z0-9_-}, 1-64 chars).",
            username,
        )
        return
    try:
        admin_id = user_store.create(username, BOOTSTRAP_ADMIN_PASSWORD, is_admin=True, scheme=PASSWORD_HASH_SCHEME)
        provision_user_storage(admin_id)
        _seed_admin_integrations_from_env(admin_id)
    except Exception:
        LOGGER.exception("failed to bootstrap admin user %r", username)
        return
    if BOOTSTRAP_ADMIN_PASSWORD == _DEFAULT_ADMIN_PASSWORD:
        LOGGER.warning(
            "Bootstrapped admin %r with the DEFAULT password. Set LECTIO_ADMIN_PASSWORD "
            "(and change the account password) before exposing this instance.",
            username,
        )
    else:
        LOGGER.info("Bootstrapped admin user %r.", username)


def _rebuild_email_batch_queue(conn: sqlite3.Connection) -> None:
    """Swap email_batch_queue to the wider UNIQUE key, keeping rows and ids (the caller owns the transaction)."""
    conn.execute("ALTER TABLE email_batch_queue RENAME TO email_batch_queue_old")
    conn.execute(
        """
        CREATE TABLE email_batch_queue (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            rule_scope TEXT NOT NULL,
            rule_scope_id TEXT NOT NULL,
            rule_keyword TEXT NOT NULL,
            queued_at TEXT NOT NULL,
            feed_url TEXT NOT NULL,
            entry_id TEXT NOT NULL,
            title TEXT,
            link TEXT,
            feed_title TEXT,
            excerpt TEXT,
            email_to TEXT NOT NULL,
            cc_me INTEGER NOT NULL DEFAULT 0,
            rule_uid TEXT NOT NULL DEFAULT '',
            UNIQUE(rule_scope, rule_scope_id, rule_keyword, rule_uid, feed_url, entry_id)
        )
        """
    )
    conn.execute(
        "INSERT INTO email_batch_queue (id, rule_scope, rule_scope_id, rule_keyword, rule_uid, queued_at, feed_url, entry_id,"
        " title, link, feed_title, excerpt, email_to, cc_me)"
        " SELECT id, rule_scope, rule_scope_id, rule_keyword, rule_uid, queued_at, feed_url, entry_id,"
        " title, link, feed_title, excerpt, email_to, cc_me FROM email_batch_queue_old ORDER BY id"
    )
    conn.execute("DROP TABLE email_batch_queue_old")


def ensure_meta_schema() -> None:
    with get_meta_connection() as conn:
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS folders (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                name TEXT NOT NULL,
                parent_id INTEGER,
                UNIQUE(name, parent_id),
                FOREIGN KEY(parent_id) REFERENCES folders(id)
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS folder_feeds (
                folder_id INTEGER NOT NULL,
                feed_url TEXT NOT NULL,
                PRIMARY KEY(folder_id, feed_url),
                FOREIGN KEY(folder_id) REFERENCES folders(id)
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS app_settings (
                key TEXT PRIMARY KEY,
                value TEXT NOT NULL
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS saved_entries (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                saved_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                PRIMARY KEY(feed_url, entry_id)
            )
            """
        )
        # A short-lived undo for /entries/saved's unstar (POST /entries/saved,
        # saved=0) — the same shape as entry_unread_batch, one row per entry
        # keyed by a shared timestamp token that IS the undo token, but this
        # one also keeps the original saved_at so a restored star lands back
        # in its actual star-order position instead of jumping to "just
        # starred." Raised 2026-08-23: repeat-pressing the star-toggle key by
        # accident unstarred ~16 articles with no way to identify which ones
        # afterward. Deliberately scoped to this one route -- the bulk/
        # administrative unstar paths (Archive, dedup/merge, unsubscribe) go
        # through apply_star_state directly and don't write this table, since
        # an undo toast doesn't make sense for a deliberate bulk action.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS entry_unstar_batch (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                unstarred_at TEXT NOT NULL,
                saved_at TEXT,
                PRIMARY KEY (feed_url, entry_id)
            )
            """
        )
        # Mirrors entry_read_state in the other direction: a bulk mark-as-UNREAD
        # stamps its whole batch with one timestamp so the toast can put exactly
        # that batch back. entry_read_state cannot serve — its rows mean "read
        # at", and this batch has just stopped being read (they are deleted).
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS entry_unread_batch (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                unread_at TEXT NOT NULL,
                PRIMARY KEY (feed_url, entry_id)
            )
            """
        )
        # Feed-tag suggestion chips the user has dismissed, per (feed, tag).
        #
        # Manual rather than heuristic on purpose: two automatic rules were tried
        # and both hid tags that were wanted. "VinylDeals" (a place) is noise while
        # "Lessons" (a kind of content) is exactly the right filing tag, and nothing
        # in the feed metadata distinguishes them — see get_feed_tag_suggestions.
        # Scoped per feed because a tag useless on one feed can matter on another.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS suppressed_feed_tags (
                feed_url TEXT NOT NULL,
                tag TEXT NOT NULL,
                suppressed_at REAL NOT NULL,
                PRIMARY KEY (feed_url, tag)
            )
            """
        )
        # The opposite scope from suppressed_feed_tags above: a tag value that
        # should never render as a suggestion chip on ANY feed (e.g. "comments"),
        # not one dismissal per feed it happens to show up on. Filters the chip
        # out of the suggestion UI itself -- explicitly not a rule that acts on
        # entries carrying the tag, which is what tag_filter rules are for.
        # Editable from Settings -> Tags.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS suppressed_feed_tags_global (
                tag TEXT PRIMARY KEY,
                suppressed_at REAL NOT NULL
            )
            """
        )
        # Extension chips dismissed from the "This feed links:" suggestion row. The scanner cannot tell a
        # file type from a TLD lookalike — ".il" is Israel, but ".zip" and ".mov" are both real TLDs AND real
        # file types, so no fixed blocklist gets this right. Let the user say so instead. Per feed, matching
        # suppressed_feed_tags: a type that is noise on one feed can be the point on another.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS suppressed_feed_attachment_exts (
                feed_url TEXT NOT NULL,
                ext TEXT NOT NULL,
                suppressed_at REAL NOT NULL,
                PRIMARY KEY (feed_url, ext)
            )
            """
        )
        # Tag aliases: one spelling of a tag folded into another. Publishers
        # disagree about the same subject — the live library carries cpp (204)
        # beside c++ (181), csharp (149) beside c# (1), dotnet (108) beside
        # .net (3) — so filtering on one silently misses the other half.
        # Applied in normalize_tag_value, which every tag path already goes
        # through, so one row covers manual tags, captured feed tags, filter
        # rules and imports alike.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS tag_aliases (
                alias TEXT PRIMARY KEY,
                canonical TEXT NOT NULL,
                created_at REAL NOT NULL
            )
            """
        )
        # Duplicate-scan groups the user has explicitly said aren't dupes
        # (reported 2026-08-10: "Need a way to mark as Not Dupes, don't
        # suggest"). Keyed by the exact set of feed URLs shown in the group
        # at dismiss time -- if any of those feeds later drops out of the
        # scan (unsubscribed, url changed), the dismissal naturally stops
        # matching rather than silently hiding some other, unrelated group.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS dedup_dismissed (
                dismiss_key TEXT PRIMARY KEY,
                dismissed_at TEXT NOT NULL
            )
            """
        )
        # The feed-level equivalent of dedup_dismissed: a genuine user-initiated
        # unsubscribe, so an Inoreader resync's subscriptions phase (which blindly
        # re-adds anything Ino still lists as subscribed but reader is missing)
        # can tell "deliberately removed" apart from "gone for some other reason"
        # instead of resurrecting it. Written only from the actual unsubscribe
        # paths -- not dedup/merge/format-upgrade (curation already migrates
        # there) and not the YouTube-sync auto-removal callback (that one should
        # stay free to re-add a channel the user re-subscribes to on YouTube).
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS declined_feeds (
                feed_url TEXT PRIMARY KEY,
                declined_at TEXT NOT NULL
            )
            """
        )
        # The read-later "done" axis, deliberately its own table rather than a
        # column on saved_entries. A saved_entries row *is* the star, and the
        # star is the TODO axis: "I still have to decide what to do with this."
        # Done is independent — Archive removes the star but the item stays
        # archived, so the state cannot live on a row that archiving deletes.
        # NOT the offline "starred archive" (archived_entry), which is content.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS archived_entries (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                archived_at TIMESTAMP NOT NULL,
                PRIMARY KEY(feed_url, entry_id)
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS deleted_entries (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                deleted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                PRIMARY KEY(feed_url, entry_id)
            )
            """
        )
        # Manual tags for orphan-archive entries — entries whose feed is no
        # longer in `reader` (see _build_orphan_entry_detail), so there is no
        # reader resource for reader's own entry_tags to attach to. Star
        # already works for orphans (saved_entries is meta-DB, no reader
        # dependency); this gives tags the same independence rather than
        # silently no-oping when reader.get_entry() misses.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS orphan_entry_tags (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                tag TEXT NOT NULL,
                PRIMARY KEY(feed_url, entry_id, tag)
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS entry_date_overrides (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                published TEXT NOT NULL,
                source TEXT NOT NULL DEFAULT 'manual',
                PRIMARY KEY(feed_url, entry_id)
            )
            """
        )
        try:
            # Existing rows predate this column and cannot be told apart after
            # the fact, so they default to 'manual' — the safe direction, since
            # scripts/backfill_url_inferred_dates.py's --refresh only recomputes
            # rows marked 'inferred' (its own writes) and must never touch a
            # real correction made through /entries/set-date.
            conn.execute("ALTER TABLE entry_date_overrides ADD COLUMN source TEXT NOT NULL DEFAULT 'manual'")
        except Exception:
            pass  # column already exists
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS entry_title_overrides (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                title TEXT NOT NULL,
                PRIMARY KEY(feed_url, entry_id)
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS entry_link_overrides (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                link TEXT NOT NULL,
                PRIMARY KEY(feed_url, entry_id)
            )
            """
        )
        # Re-fetched content for a STARRED feed entry (whose feed content lacked
        # images or was truncated). Pinned here and reapplied after refresh, so
        # the feed re-serving its own thinner content can't clobber the fuller
        # copy the user deliberately pulled. Mirrors entry_link/title/date
        # overrides. Content is reader's JSON content shape.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS entry_content_overrides (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                content TEXT NOT NULL,
                PRIMARY KEY(feed_url, entry_id)
            )
            """
        )
        # Aardvark-style per-entry cleanup: the pristine body (reader's JSON
        # content shape) plus the ops that were replayed over it. The original
        # is what "Revert cleanup" restores; the ops are the record of *what*
        # was removed, which a later per-feed rule gets promoted from. Only the
        # first edit writes original_content — repeated cleanups of the same
        # entry must keep reverting to the true original, not to the last edit.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS entry_content_edits (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                original_content TEXT NOT NULL,
                ops TEXT NOT NULL,
                edited_at TEXT NOT NULL,
                PRIMARY KEY(feed_url, entry_id)
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS entry_read_state (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                read_at TEXT NOT NULL,
                PRIMARY KEY(feed_url, entry_id)
            )
            """
        )
        # "Fix URLs" automation: per-feed host aliases. When an author moves
        # domains without updating their feed's <guid>/<link> (they keep emitting
        # the old host), every entry lands with an old-domain id/link. A rule
        # here rewrites from_host -> to_host at ingest so entries arrive with the
        # current domain. Per-feed by design: an author's dead domains are only
        # rewritten inside their own feed, never a legit link elsewhere.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS feed_url_rewrites (
                feed_url TEXT NOT NULL,
                from_host TEXT NOT NULL,
                to_host TEXT NOT NULL,
                PRIMARY KEY(feed_url, from_host)
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS read_history (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                title TEXT,
                link TEXT,
                feed_title TEXT,
                read_at TEXT NOT NULL,
                UNIQUE(feed_url, entry_id)
            )
            """
        )
        conn.execute("CREATE INDEX IF NOT EXISTS idx_read_history_read_at ON read_history(read_at DESC)")
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS youtube_video_duration (
                video_id TEXT PRIMARY KEY,
                duration_seconds INTEGER,
                duration_display TEXT,
                fetched_at TEXT NOT NULL DEFAULT (datetime('now'))
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS feed_failure_state (
                feed_url TEXT PRIMARY KEY,
                consecutive_failures INTEGER NOT NULL DEFAULT 0,
                next_retry_at REAL,
                last_error TEXT,
                last_failure_at REAL,
                last_success_at REAL
            )
            """
        )
        try:
            conn.execute("ALTER TABLE feed_failure_state ADD COLUMN acknowledged_at REAL")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE folders ADD COLUMN cadence_minutes INTEGER DEFAULT NULL")
        except Exception:
            pass
        # Per-folder retention: delete read posts N days after they were read
        # (NULL/0 = keep forever). Applied by the nightly maintenance prune.
        try:
            conn.execute("ALTER TABLE folders ADD COLUMN retention_days INTEGER DEFAULT NULL")
        except Exception:
            pass
        # The entry ids each feed served in its most recent parse. Tomb
        # sweeping refuses to drop tombstones still present here: a quiet
        # YouTube channel serves the same 15 videos for years, and sweeping
        # their tombstones would resurrect all of them on the channel's next
        # upload (the reparse re-ingests the whole window).
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS feed_seen_window (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                PRIMARY KEY (feed_url, entry_id)
            )
            """
        )
        # Tombstone age for the nightly sweep. Pre-existing rows (and rows
        # written by old code during a rolling deploy) are backfilled to "now"
        # so they age out one sweep-window from the upgrade, not immediately.
        try:
            conn.execute("ALTER TABLE deleted_entries ADD COLUMN created_at TIMESTAMP DEFAULT NULL")
        except Exception:
            pass
        try:
            conn.execute(
                "UPDATE deleted_entries SET created_at = ? WHERE created_at IS NULL",
                (datetime.now().isoformat(),),
            )
        except Exception:
            pass
        # Read Mode "Archive" state: originally a per-saved-item flag (NULL =
        # inbox). Superseded by the archived_entries table above — see its
        # comment for why the done-axis cannot live on the star row. The column
        # is still created (old rows carry it, and it is the migration source)
        # but nothing reads it after the backfill below.
        try:
            conn.execute("ALTER TABLE saved_entries ADD COLUMN archived_at TIMESTAMP DEFAULT NULL")
        except Exception:
            pass
        # One-time lift of the legacy column into the new table — and it must be
        # ONE-TIME, not merely idempotent. The first version relied on INSERT OR
        # IGNORE and left the column populated, so every boot re-lifted it:
        # un-archiving deleted the archived_entries row, the old column survived,
        # and the next restart brought the item back. Reported as "I've
        # unarchived them 3 times now and they keep coming back" — a resurrection
        # bug that is invisible until something restarts the app.
        #
        # Clearing the source in the same transaction is what makes it one-time,
        # and it also removes the second source of truth. A marker flag would
        # leave a populated column that still looks authoritative to the next
        # person reading the schema.
        try:
            conn.execute(
                "INSERT OR IGNORE INTO archived_entries (feed_url, entry_id, archived_at) "
                "SELECT feed_url, entry_id, archived_at FROM saved_entries "
                "WHERE archived_at IS NOT NULL"
            )
            conn.execute("UPDATE saved_entries SET archived_at = NULL WHERE archived_at IS NOT NULL")
            conn.commit()
        except Exception:
            pass
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS entry_lead_images (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                image_url TEXT,
                fetched_at REAL NOT NULL,
                PRIMARY KEY (feed_url, entry_id)
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS entry_feed_tags (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                tag TEXT NOT NULL,
                first_seen_at REAL NOT NULL,
                PRIMARY KEY (feed_url, entry_id, tag)
            )
            """
        )
        # Where a tag came from: 'feed' (ingest) or 'page' (page topics, services/page_topics.py). The feed's per-entry replace only
        # touches its own rows, so page topics survive the feed re-delivering an entry on every refresh.
        if "source" not in {r[1] for r in conn.execute("PRAGMA table_info(entry_feed_tags)").fetchall()}:
            conn.execute("ALTER TABLE entry_feed_tags ADD COLUMN source TEXT NOT NULL DEFAULT 'feed'")
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS domain_failure_state (
                domain TEXT PRIMARY KEY,
                consecutive_failures INTEGER NOT NULL DEFAULT 0,
                next_retry_at REAL,
                last_failure_at REAL
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS feed_lead_image_strategy (
                feed_url TEXT PRIMARY KEY,
                strategy TEXT NOT NULL DEFAULT 'unknown',
                detected_at REAL NOT NULL,
                manual INTEGER NOT NULL DEFAULT 0
            )
            """
        )
        try:
            conn.execute("ALTER TABLE feed_lead_image_strategy ADD COLUMN manual INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass  # column already exists
        try:
            conn.execute("ALTER TABLE entry_lead_images ADD COLUMN image_alt TEXT")
        except Exception:
            pass  # column already exists
        try:
            conn.execute("ALTER TABLE entry_lead_images ADD COLUMN image_title TEXT")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE entry_lead_images ADD COLUMN thumb_crop TEXT")
        except Exception:
            pass
        # A webcomic page like cad-comic.com's can mark a strip as locked to
        # paying supporters until a stated date — captured from the SAME
        # source-page fetch already made for the lead image (see
        # LeadImageService.check_and_cache_webcomic_lock), so hide_locked_comics
        # costs nothing extra to detect. NULL/absent means "not locked" or
        # "never checked"; a real value in the past means it has since unlocked,
        # so no periodic re-check job is needed — the date does the expiring.
        try:
            conn.execute("ALTER TABLE entry_lead_images ADD COLUMN locked_until REAL")
        except Exception:
            pass
        # The locked-comics lookup asks for every non-NULL row library-wide; a handful, but a full scan of ~150k rows (~55ms) without this.
        conn.execute(
            "CREATE INDEX IF NOT EXISTS idx_entry_lead_images_locked ON entry_lead_images(feed_url) WHERE locked_until IS NOT NULL"
        )
        try:
            conn.execute("ALTER TABLE feed_strategy_cache ADD COLUMN image_alt TEXT")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE feed_strategy_cache ADD COLUMN image_title TEXT")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN caption_source TEXT")
        except Exception:
            pass
        # "<param>=<value>", e.g. "w=1600". See upgrade_image_size_param.
        try:
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN image_size_rule TEXT")
        except Exception:
            pass
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS disabled_feeds (
                feed_url TEXT PRIMARY KEY,
                disabled_at TEXT NOT NULL DEFAULT (datetime('now'))
            )
            """
        )
        # Why a feed is disabled: '' = the user paused it; anything else is the system's own reason ("410 Gone", ...). Rows that
        # predate the column are the user's, which is what the default says.
        try:
            conn.execute("ALTER TABLE disabled_feeds ADD COLUMN reason TEXT NOT NULL DEFAULT ''")
        except Exception:
            pass  # column already exists
        # Feeds unsubscribed-but-retained for their curation (starred/tagged
        # entries). Kept feeds are hidden from the feed tree / All Feeds /
        # Uncategorized and have updates disabled, but their reader feed and
        # entries survive so the Saved/Kept view can still browse them per feed.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS kept_feeds (
                feed_url TEXT PRIMARY KEY,
                kept_at TEXT NOT NULL DEFAULT (datetime('now'))
            )
            """
        )
        # Hosts the user has marked as never having a feed, so the saved-article
        # auto-filer stops proposing them. Their saves are genuine one-off
        # read-later captures — a cheat sheet, a one-time tutorial — not unfiled
        # feed articles, and without this they reappear in every scan pass and
        # never resolve.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS autofile_non_feed_hosts (
                host TEXT PRIMARY KEY,
                marked_at TEXT NOT NULL DEFAULT (datetime('now'))
            )
            """
        )
        # Subscriptions that aren't really feeds — a single article URL that got
        # added as one, a scraped page. They stay subscribed (their entries are
        # real saved reading) but must never be offered as a destination for
        # filing, or the site's whole backlog lands in a one-article stub.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS non_feed_subscriptions (
                feed_url TEXT PRIMARY KEY,
                marked_at TEXT NOT NULL DEFAULT (datetime('now'))
            )
            """
        )
        # Dead feeds the user has triaged as "needs a replacement": updates are
        # disabled (we stop hitting them) and they move out of the Failing list
        # into a Needs-replacement worklist, but the feed + its entries/curation
        # are kept so a Change-URL swap can reattach a working source later.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS feeds_needing_replacement (
                feed_url TEXT PRIMARY KEY,
                flagged_at TEXT NOT NULL DEFAULT (datetime('now'))
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS browser_ua_feeds (
                feed_url TEXT PRIMARY KEY,
                flagged_at TEXT NOT NULL DEFAULT (datetime('now')),
                reason TEXT NOT NULL DEFAULT ''
            )
            """
        )
        # as_needed proxy escalation: a feed lands here only after browser-UA
        # was already in play (flagged this cycle or earlier) and the fetch
        # still failed — see _flag_proxy_feed_on_still_blocked. Mode-gated:
        # only as_needed users ever get a feed flagged here (off never wants
        # it, always doesn't need per-feed tracking).
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS proxy_feeds (
                feed_url TEXT PRIMARY KEY,
                flagged_at TEXT NOT NULL DEFAULT (datetime('now')),
                reason TEXT NOT NULL DEFAULT ''
            )
            """
        )
        # Last-resort escalation, one rung past proxy_feeds: a feed lands here
        # only after the primary proxy was already in play and still failed —
        # see _flag_tailscale_feed_on_still_blocked. Same mode-gating as
        # proxy_feeds, plus requires SETTING_TAILSCALE_URL configured.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS tailscale_feeds (
                feed_url TEXT PRIMARY KEY,
                flagged_at TEXT NOT NULL DEFAULT (datetime('now')),
                reason TEXT NOT NULL DEFAULT ''
            )
            """
        )
        # FlareSolverr escalation, between proxy_feeds and tailscale_feeds: a
        # feed lands here only after the primary proxy was already in play and
        # the fetch is STILL specifically a bot-challenge (not any refusal) —
        # see _flag_flaresolverr_feed_on_still_blocked.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS flaresolverr_feeds (
                feed_url TEXT PRIMARY KEY,
                flagged_at TEXT NOT NULL DEFAULT (datetime('now')),
                reason TEXT NOT NULL DEFAULT ''
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS feed_display_prefs (
                feed_url TEXT PRIMARY KEY,
                show_lead_image_in_article INTEGER NOT NULL DEFAULT 1,
                show_lead_image_as_thumb INTEGER NOT NULL DEFAULT 1,
                show_image_caption INTEGER NOT NULL DEFAULT -1,
                caption_source TEXT,
                inject_source_images INTEGER NOT NULL DEFAULT 0
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS feed_strategy_cache (
                feed_url TEXT NOT NULL,
                strategy TEXT NOT NULL,
                image_url TEXT,
                fetched_at REAL NOT NULL,
                error TEXT,
                image_alt TEXT,
                image_title TEXT,
                PRIMARY KEY (feed_url, strategy)
            )
            """
        )
        # Per-refresh fetch history — one row per non-skipped refresh attempt, so
        # Feed Properties can show why a feed is stale or flagged problematic.
        # Bounded by a per-feed cap + age prune in daily maintenance.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS feed_fetch_history (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                feed_url TEXT NOT NULL,
                fetched_at REAL NOT NULL,
                status TEXT NOT NULL,
                http_status INTEGER,
                new_entries INTEGER,
                duration_ms INTEGER,
                error TEXT
            )
            """
        )
        conn.execute("CREATE INDEX IF NOT EXISTS feed_fetch_history_by_feed ON feed_fetch_history (feed_url, fetched_at DESC)")
        # Recovered media:content podcast audio (reader drops media:content, so we
        # re-parse the raw feed on demand). entry_media_audio holds per-entry audio
        # URLs found; feed_media_scan tracks when a feed was last scanned (and
        # whether any audio was found) so we don't re-fetch non-podcast feeds.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS entry_media_audio (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                audio_url TEXT NOT NULL,
                PRIMARY KEY (feed_url, entry_id)
            )
            """
        )
        # Recovered YouTube embeds (feedparser strips the <iframe>, so a WP
        # YouTube block reaches us as an empty figure). video_ids is a
        # space-separated list in document order; '' means a YouTube embed was
        # present but no id was recoverable (a cached negative). Populated by the
        # same raw-feed re-parse as entry_media_audio.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS entry_media_video (
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                video_ids TEXT NOT NULL DEFAULT '',
                PRIMARY KEY (feed_url, entry_id)
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS feed_media_scan (
                feed_url TEXT PRIMARY KEY,
                scanned_at REAL NOT NULL,
                found INTEGER NOT NULL DEFAULT 0,
                suggested_audio_feed TEXT NOT NULL DEFAULT '',
                ok INTEGER NOT NULL DEFAULT 1
            )
            """
        )
        # Added after feed_media_scan shipped: the podcast-host feed discovered
        # for an audio-less feed (so we can suggest subscribing to the audio feed).
        try:
            conn.execute("ALTER TABLE feed_media_scan ADD COLUMN suggested_audio_feed TEXT NOT NULL DEFAULT ''")
        except Exception:
            pass
        # Added later: distinguish a scan that completed cleanly with no audio
        # (back off a long time) from one that errored mid-way — e.g. a host feed
        # we discovered but couldn't fetch (Cloudflare 403) — so transient
        # failures retry soon instead of inheriting the 7-day "empty" backoff.
        try:
            conn.execute("ALTER TABLE feed_media_scan ADD COLUMN ok INTEGER NOT NULL DEFAULT 1")
        except Exception:
            pass
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS highlight_keywords (
                scope TEXT NOT NULL,
                scope_id TEXT NOT NULL,
                keyword TEXT NOT NULL,
                color TEXT NOT NULL DEFAULT 'yellow',
                PRIMARY KEY (scope, scope_id, keyword)
            )
            """
        )
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN is_regex INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN enabled INTEGER NOT NULL DEFAULT 1")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN type TEXT NOT NULL DEFAULT 'highlight'")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN search_in TEXT NOT NULL DEFAULT 'title'")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN delivery TEXT NOT NULL DEFAULT 'immediately'")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN email_to TEXT NOT NULL DEFAULT ''")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN batch_time TEXT NOT NULL DEFAULT ''")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN batch_count INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN cc_me INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN dedup_window_hours INTEGER NOT NULL DEFAULT 24")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN exclude_scope_ids TEXT NOT NULL DEFAULT ''")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN dedup_fuzzy_pct INTEGER NOT NULL DEFAULT 80")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN dedup_min_title_words INTEGER NOT NULL DEFAULT 5")
        except Exception:
            pass
        # The column shipped pre-filled with 4, so no rule holding 4 had it chosen
        # by anyone. Raise those once — 5 drops two same-feed false collisions and
        # loses no real duplicate. Flagged so a deliberate 4 set later survives.
        try:
            if get_setting(conn, "dedup_min_title_words_bumped_to_5") != "1":
                conn.execute(
                    "UPDATE highlight_keywords SET dedup_min_title_words = 5 WHERE type = 'deduplicate' AND dedup_min_title_words = 4"
                )
                set_setting(conn, "dedup_min_title_words_bumped_to_5", "1")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN sort_order INTEGER NOT NULL DEFAULT 0")
            conn.execute("UPDATE highlight_keywords SET sort_order = rowid WHERE sort_order = 0")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN webhook_url TEXT NOT NULL DEFAULT ''")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN webhook_format TEXT NOT NULL DEFAULT 'generic'")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN webhook_batch INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass
        # youtube_playlist rule type: target playlist + behavior toggles.
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN yt_playlist_id TEXT NOT NULL DEFAULT ''")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN yt_playlist_title TEXT NOT NULL DEFAULT ''")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN yt_include_shorts INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN yt_mark_read INTEGER NOT NULL DEFAULT 1")
        except Exception:
            pass
        # Duration filter for youtube_playlist (minutes; 0 = no limit).
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN yt_min_minutes INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN yt_max_minutes INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass
        # A rule's identity is (scope, scope_id, keyword) everywhere in this file --
        # fine for same-request lookups, but any table that needs to recognize "the
        # same rule" *across time* breaks the moment scope/scope_id/keyword changes
        # (editing a rule's feed list, say): edit_highlight_route does remove-old +
        # insert-new on an identity change, so anything keyed on the old text is
        # silently orphaned. rule_uid is a stable id that survives such edits --
        # add_highlight_keyword carries the prior row's rule_uid forward instead of
        # minting a new one. Existing rows get one backfilled below (once; empty
        # rule_uid afterward means a row created before this migration ran and
        # somehow missed the backfill, not "no rule_uid assigned yet").
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN rule_uid TEXT NOT NULL DEFAULT ''")
        except Exception:
            pass
        for row in conn.execute("SELECT rowid FROM highlight_keywords WHERE rule_uid = ''").fetchall():
            conn.execute(
                "UPDATE highlight_keywords SET rule_uid = ? WHERE rowid = ?",
                (secrets.token_hex(16), row["rowid"]),
            )
        # Optional display name, purely cosmetic (never matched against, never
        # part of the (scope, scope_id, keyword) identity) -- for a MAR rule
        # whose keyword has grown into a long regex, the rules list can show
        # this instead, with the pattern still one hover away as a tooltip.
        try:
            conn.execute("ALTER TABLE highlight_keywords ADD COLUMN label TEXT NOT NULL DEFAULT ''")
        except Exception:
            pass
        # Dedup guard for the youtube_playlist rule: playlistItems.insert is not
        # idempotent, so record each (rule, entry) we've added to avoid re-adding the
        # same video on a later refresh (the cutoff window alone can re-match).
        # Keyed on (scope, scope_id, keyword) for historical reasons; that text
        # identity breaks on a scope-changing edit (see rule_uid above), so new
        # rows also carry rule_uid and get deduped through the partial unique index
        # below instead once it's populated. Old rows keep rule_uid = '' rather
        # than get backfilled: there's no reliable way to map them back to a
        # *current* rule now that their scope text may no longer match anything.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS youtube_playlist_added (
                scope TEXT NOT NULL,
                scope_id TEXT NOT NULL DEFAULT '',
                keyword TEXT NOT NULL DEFAULT '',
                entry_id TEXT NOT NULL,
                video_id TEXT NOT NULL,
                added_at TEXT NOT NULL,
                PRIMARY KEY (scope, scope_id, keyword, entry_id, video_id)
            )
            """
        )
        try:
            conn.execute("ALTER TABLE youtube_playlist_added ADD COLUMN rule_uid TEXT NOT NULL DEFAULT ''")
        except Exception:
            pass
        conn.execute(
            "CREATE UNIQUE INDEX IF NOT EXISTS idx_ytpl_added_rule_uid"
            " ON youtube_playlist_added (rule_uid, entry_id, video_id) WHERE rule_uid != ''"
        )
        # Per-user YouTube Data API quota spend, keyed by the Pacific calendar date
        # (Google resets quota at midnight Pacific). The API exposes no remaining-quota
        # read, so we estimate by summing each call's documented unit cost.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS yt_quota_spend (
                day TEXT PRIMARY KEY,
                units INTEGER NOT NULL DEFAULT 0
            )
            """
        )
        # Quire API call log — one row per billed call, used for the sliding-window
        # rate meter (Quire limits per org by minute + hour, with no remaining-quota
        # read). Rows older than an hour are pruned on each insert.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS quire_call_log (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                ts INTEGER NOT NULL
            )
            """
        )
        conn.execute("CREATE INDEX IF NOT EXISTS idx_quire_call_log_ts ON quire_call_log (ts)")
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS rule_run_log (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                run_at TEXT NOT NULL,
                rule_type TEXT NOT NULL,
                scope TEXT NOT NULL,
                scope_id TEXT NOT NULL DEFAULT '',
                keyword TEXT NOT NULL DEFAULT '',
                entries_affected INTEGER NOT NULL DEFAULT 0,
                trigger TEXT NOT NULL DEFAULT 'manual'
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS rule_run_log_entries (
                log_id   INTEGER NOT NULL,
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                title    TEXT,
                link     TEXT,
                feed_title TEXT,
                role     TEXT NOT NULL DEFAULT 'marked'
            )
            """
        )
        conn.execute("CREATE INDEX IF NOT EXISTS rule_run_log_entries_log_id ON rule_run_log_entries (log_id)")
        try:
            # 'marked' = entry was marked read; 'kept' = the surviving copy of a
            # dedup group, logged so run history shows what each duplicate matched.
            conn.execute("ALTER TABLE rule_run_log_entries ADD COLUMN role TEXT NOT NULL DEFAULT 'marked'")
        except sqlite3.OperationalError as exc:
            if "duplicate column" not in str(exc).lower():
                raise
        try:
            # The kept copy a marked duplicate matched (its link), so dedup run
            # history can pair each marked entry with its keeper for verification
            # instead of listing all marked then all kept. NULL for non-dedup runs
            # and for kept rows (which anchor their own group by their own link).
            conn.execute("ALTER TABLE rule_run_log_entries ADD COLUMN matched_link TEXT")
        except sqlite3.OperationalError as exc:
            if "duplicate column" not in str(exc).lower():
                raise
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS dedup_false_matches (
                keep_link TEXT NOT NULL,
                mark_link TEXT NOT NULL,
                added_at  TEXT NOT NULL,
                PRIMARY KEY (keep_link, mark_link)
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS email_contacts (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                label TEXT NOT NULL,
                address TEXT NOT NULL UNIQUE
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS email_batch_queue (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                rule_scope TEXT NOT NULL,
                rule_scope_id TEXT NOT NULL,
                rule_keyword TEXT NOT NULL,
                queued_at TEXT NOT NULL,
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                title TEXT,
                link TEXT,
                feed_title TEXT,
                excerpt TEXT,
                email_to TEXT NOT NULL,
                cc_me INTEGER NOT NULL DEFAULT 0,
                rule_uid TEXT NOT NULL DEFAULT '',
                UNIQUE(rule_scope, rule_scope_id, rule_keyword, rule_uid, feed_url, entry_id)
            )
            """
        )
        # Same scope-text-identity fragility rule_uid was added for above
        # (highlight_keywords / youtube_playlist_added): editing a batch email
        # rule's scope/keyword while entries are queued splits its backlog
        # across two text identities instead of following the same rule. New
        # rows also carry rule_uid; old rows keep '' for the same reason
        # youtube_playlist_added's do -- there is no reliable way to map them
        # back to a *current* rule once their scope text may no longer match
        # anything. Flush paths prefer rule_uid when present and fall back to
        # the text tuple otherwise, same shape as the dedup index below it.
        try:
            conn.execute("ALTER TABLE email_batch_queue ADD COLUMN rule_uid TEXT NOT NULL DEFAULT ''")
        except Exception:
            pass
        # The original UNIQUE key omitted feed_url and rule_uid, so two rules sharing scope and keyword text (different
        # recipients), or one entry_id in two feeds, collided and the second INSERT OR IGNORE was dropped. A UNIQUE
        # constraint can't be altered in place: rebuild the table when the stored DDL still has the old key.
        _bq_ddl = conn.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name='email_batch_queue'").fetchone()
        if _bq_ddl and "rule_keyword, entry_id)" in (_bq_ddl[0] or ""):
            # Python's sqlite3 doesn't open a transaction before DDL, so make one explicit: a failure mid-rebuild must roll back
            # the rename and create too, or the queued rows are stranded in the _old table and detection never fires again.
            if conn.in_transaction:
                conn.commit()
            conn.execute("BEGIN IMMEDIATE")
            try:
                _rebuild_email_batch_queue(conn)
            except BaseException:
                conn.rollback()
                raise
            conn.commit()
        # Per-rule, per-entry claim for the webhook / Quire / immediate-email dispatchers: they select entries "added in the last
        # 15 minutes", so two passes for one feed inside that window would otherwise fire a match twice. A row is inserted before
        # the send and deleted if the send fails (batch-mode email claims with the queue insert, never released);
        # rows past the window are pruned nightly.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS automation_dispatched (
                rule_key TEXT NOT NULL,
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                dispatched_at TEXT NOT NULL,
                PRIMARY KEY (rule_key, feed_url, entry_id)
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS scraped_feeds (
                id TEXT PRIMARY KEY,
                source_url TEXT NOT NULL,
                mode TEXT NOT NULL,
                selector TEXT,
                feed_title TEXT NOT NULL,
                created_at TEXT NOT NULL,
                last_scraped_at TEXT,
                last_content_hash TEXT
            )
            """
        )
        # link_list only: a CSS selector applied to each NEW entry's own page
        # (not the listing page `selector` already scopes) to fill its body.
        # Readability/full-page guessing both fail on a chrome-heavy listing
        # site the same way -- confirmed live 2026-09-19 on texasbluesalley.com,
        # where the real content is a small, easily-selected region but the
        # site's own nav/header markup is large enough to win readability's
        # size-based scoring. See docs/architecture/feeds.md.
        try:
            conn.execute("ALTER TABLE scraped_feeds ADD COLUMN content_selector TEXT")
        except Exception:
            pass
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS scraped_entries (
                id TEXT PRIMARY KEY,
                scraped_feed_id TEXT NOT NULL REFERENCES scraped_feeds(id),
                entry_url TEXT,
                title TEXT NOT NULL,
                content TEXT,
                published_at TEXT NOT NULL,
                hidden INTEGER NOT NULL DEFAULT 0,
                UNIQUE(scraped_feed_id, entry_url)
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS deviantart_feeds (
                id TEXT PRIMARY KEY,
                username TEXT NOT NULL,
                feed_title TEXT NOT NULL,
                created_at TEXT NOT NULL,
                last_synced_at TEXT
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS deviantart_entries (
                id TEXT PRIMARY KEY,
                deviantart_feed_id TEXT NOT NULL REFERENCES deviantart_feeds(id),
                deviationid TEXT NOT NULL,
                title TEXT NOT NULL,
                entry_url TEXT,
                content TEXT,
                published_at TEXT NOT NULL,
                UNIQUE(deviantart_feed_id, deviationid)
            )
            """
        )
        # Watched artists whose DeviantArt account is deactivated ("Account is
        # inactive."). Parked here so the watch-list sync stops retrying them on
        # every run (they can never be added), and daily maintenance re-checks
        # for reactivation.
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS deviantart_deactivated (
                username TEXT PRIMARY KEY,
                first_seen_at TEXT NOT NULL,
                last_checked_at TEXT NOT NULL
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS devto_feeds (
                id TEXT PRIMARY KEY,
                feed_title TEXT NOT NULL,
                tag TEXT,
                top_days INTEGER,
                english_only INTEGER NOT NULL DEFAULT 1,
                min_reactions INTEGER,
                tags_exclude TEXT,
                created_at TEXT NOT NULL,
                last_synced_at TEXT
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS devto_entries (
                id TEXT PRIMARY KEY,
                devto_feed_id TEXT NOT NULL REFERENCES devto_feeds(id),
                article_id TEXT NOT NULL,
                title TEXT NOT NULL,
                entry_url TEXT,
                content TEXT,
                published_at TEXT NOT NULL,
                UNIQUE(devto_feed_id, article_id)
            )
            """
        )
        # Comma-joined tag slugs; the synthetic RSS is regenerated from these
        # rows, so tags must persist here to come out as <category> elements.
        try:
            conn.execute("ALTER TABLE devto_entries ADD COLUMN tags TEXT NOT NULL DEFAULT ''")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE deviantart_entries ADD COLUMN tags TEXT NOT NULL DEFAULT ''")
        except Exception:
            pass
        # When the /deviation/metadata lookup ran for this entry (tags aren't in
        # browse/gallery responses). Set even when the result is zero tags, so
        # untagged deviations aren't re-looked-up on every refresh.
        try:
            conn.execute("ALTER TABLE deviantart_entries ADD COLUMN tags_fetched_at TEXT")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE deviantart_entries ADD COLUMN author TEXT NOT NULL DEFAULT ''")
        except Exception:
            pass
        # Set even when the metadata lookup came back with no author (mirrors
        # tags_fetched_at above), so an entry the API never attributes isn't
        # re-looked-up on every refresh forever.
        try:
            conn.execute("ALTER TABLE deviantart_entries ADD COLUMN author_fetched_at TEXT")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE deviantart_feeds ADD COLUMN source TEXT NOT NULL DEFAULT 'gallery'")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN hide_shorts INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN hide_unpremiered INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass
        # Members-only videos: detected via the "Members only" badge on the
        # video's own watch page (see youtube_video_duration.members_only) --
        # unlike hide_shorts/hide_unpremiered this isn't in the Data API
        # response already fetched for duration, so it costs a real per-video
        # page fetch and is opt-in like hide_paywalled rather than free like
        # its YouTube siblings.
        try:
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN hide_members_only INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass
        # Locked webcomic strips: a publisher (cad-comic.com and the like) can
        # gate a strip behind a supporter tier until a stated date. Not
        # YouTube-specific — see LeadImageService.check_and_cache_webcomic_lock.
        try:
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN hide_locked_comics INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass
        # Paywalled/subscriber-only posts: a Substack paid post ships a body that
        # is nothing but a "Read more" link back to itself, so it can be spotted
        # without any explicit marker (Substack provides none).
        try:
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN hide_paywalled INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass
        # Auto-mark posts read once older than N days (0 = no limit of its own); folders hold the same as NULL/N.
        try:
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN auto_read_days INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE folders ADD COLUMN auto_read_days INTEGER DEFAULT NULL")
        except Exception:
            pass
        # Tags this feed's posts usually want, pinned to the front of the
        # suggestion chips. A feed with a stable subject (a guitar blog: #guitar,
        # #bass) publishes no tags saying so, leaving the same words to be typed
        # on every post. Stored space-separated, normalized on write.
        try:
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN suggested_tags TEXT")
        except Exception:
            pass
        # File extensions to keep alongside a post when it is saved — guitar-pro
        # links .gp tabs and .pdf lyric sheets that vanish with the post. Stored
        # space-separated and normalized; empty/absent means the feature is off
        # for this feed.
        try:
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN attachment_exts TEXT")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN inject_source_images INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN feed_thumbnail_url TEXT")
        except Exception:
            pass
        try:
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN thumb_crop TEXT NOT NULL DEFAULT 'cover'")
        except Exception:
            pass
        # Migrate legacy 'left' value to 'cover-left'
        conn.execute("UPDATE feed_display_prefs SET thumb_crop = 'cover-left' WHERE thumb_crop = 'left'")
        # Normalize any unknown values to 'cover'
        conn.execute(
            "UPDATE feed_display_prefs SET thumb_crop = 'cover' WHERE thumb_crop NOT IN "
            "('cover','cover-top-left','cover-top','cover-top-right','cover-left','cover-right',"
            "'cover-bottom-left','cover-bottom','cover-bottom-right','contain','smart')"
        )
        try:
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN thumb_strategy TEXT")
        except Exception:
            pass
        try:
            # Per-feed SmartCrop min_scale (NULL = default 0.9); applies when thumb_crop='smart'.
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN smart_min_scale REAL")
        except Exception:
            pass
        try:
            # Per-feed Fill zoom multiplier (NULL = default 1.0); applies to cover crop modes.
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN fill_zoom REAL")
        except Exception:
            pass
        try:
            # Off by default: a bare $...$ pair is common in ordinary text
            # ("between $50 and $100") and would render as broken math on any
            # feed that isn't actually LaTeX-flavored -- see renderMathInEntryPane.
            conn.execute("ALTER TABLE feed_display_prefs ADD COLUMN katex_dollar_math INTEGER NOT NULL DEFAULT 0")
        except Exception:
            pass
        full_content_fetch.ensure_schema(conn)
        page_topics.ensure_schema(conn)
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS websub_subscriptions (
                feed_url      TEXT PRIMARY KEY,
                hub_url       TEXT,
                secret        TEXT,
                lease_seconds INTEGER DEFAULT 0,
                subscribed_at REAL    DEFAULT 0,
                expires_at    REAL    DEFAULT 0,
                verified      INTEGER DEFAULT 0,
                hub_tried_at  REAL    DEFAULT 0
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS fever_feed_map (
                id       INTEGER PRIMARY KEY AUTOINCREMENT,
                feed_url TEXT UNIQUE NOT NULL
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS fever_group_map (
                id    INTEGER PRIMARY KEY AUTOINCREMENT,
                title TEXT UNIQUE NOT NULL
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS fever_entry_map (
                id       INTEGER PRIMARY KEY AUTOINCREMENT,
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                UNIQUE(feed_url, entry_id)
            )
            """
        )
        conn.execute("CREATE INDEX IF NOT EXISTS idx_fever_entry_map_feed ON fever_entry_map(feed_url)")
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS miniflux_feed_map (
                id       INTEGER PRIMARY KEY AUTOINCREMENT,
                feed_url TEXT UNIQUE NOT NULL
            )
            """
        )
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS miniflux_entry_map (
                id       INTEGER PRIMARY KEY AUTOINCREMENT,
                feed_url TEXT NOT NULL,
                entry_id TEXT NOT NULL,
                UNIQUE(feed_url, entry_id)
            )
            """
        )
        conn.execute("CREATE INDEX IF NOT EXISTS idx_miniflux_entry_map_feed ON miniflux_entry_map(feed_url)")
        conn.execute(
            """
            CREATE TABLE IF NOT EXISTS greader_tokens (
                token      TEXT PRIMARY KEY,
                expires_at REAL NOT NULL
            )
            """
        )
        root = conn.execute(
            "SELECT id FROM folders WHERE name = ? AND parent_id IS NULL",
            (ROOT_FOLDER_NAME,),
        ).fetchone()
        if not root:
            conn.execute(
                "INSERT INTO folders (name, parent_id) VALUES (?, NULL)",
                (ROOT_FOLDER_NAME,),
            )
        if DEBUG_MODE:
            # Ensure the _Lectio system folder exists (used for dev feeds).
            root_id = conn.execute("SELECT id FROM folders WHERE name = ? AND parent_id IS NULL", (ROOT_FOLDER_NAME,)).fetchone()
            if root_id:
                conn.execute(
                    "INSERT OR IGNORE INTO folders (name, parent_id) VALUES (?, ?)",
                    (_LECTIO_FOLDER_NAME, int(root_id["id"])),
                )
        conn.execute(
            "INSERT OR IGNORE INTO app_settings (key, value) VALUES (?, ?)",
            (AUTO_REFRESH_SETTING_KEY, str(get_instance_default_auto_refresh())),
        )
        conn.execute(
            "INSERT OR IGNORE INTO app_settings (key, value) VALUES (?, ?)",
            (SORT_BY_SETTING_KEY, DEFAULT_SORT_BY),
        )
        conn.execute(
            "INSERT OR IGNORE INTO app_settings (key, value) VALUES (?, ?)",
            (SORT_DIR_SETTING_KEY, DEFAULT_SORT_DIR),
        )
        conn.execute(
            "INSERT OR IGNORE INTO app_settings (key, value) VALUES (?, ?)",
            (GLOBAL_NOTE_SETTING_KEY, ""),
        )
        pass  # email_to seeding removed; Contacts tab manages recipients


def _load_app_settings_cache(conn: sqlite3.Connection) -> dict[str, str]:
    rows = conn.execute("SELECT key, value FROM app_settings").fetchall()
    return {str(r["key"]): str(r["value"]) for r in rows}


def get_setting(conn: sqlite3.Connection, key: str) -> str | None:
    """Read a setting from the current user's in-memory cache. Loaded once per
    user on first access, kept consistent through set_setting writes."""
    uid = tenancy.current_user_id()
    with _app_settings_cache_lock:
        cache = _app_settings_cache.get(uid)
        if cache is None:
            cache = _load_app_settings_cache(conn)
            _app_settings_cache[uid] = cache
        return cache.get(key)


def set_setting(conn: sqlite3.Connection, key: str, value: str) -> None:
    """Idempotent setting write. Updates the current user's in-memory cache and
    persists only when the value actually changed."""
    uid = tenancy.current_user_id()
    with _app_settings_cache_lock:
        cache = _app_settings_cache.get(uid)
        if cache is None:
            cache = _load_app_settings_cache(conn)
            _app_settings_cache[uid] = cache
        if cache.get(key) == value:
            return
        cache[key] = value
    conn.execute(
        "INSERT OR REPLACE INTO app_settings (key, value) VALUES (?, ?)",
        (key, value),
    )


def get_cached_setting(key: str) -> str | None:
    """Read the current user's cache without a DB connection. None if unloaded or no user is bound
    (pre-auth pages, bare background threads), so callers fall through to their env/admin fallback."""
    uid = tenancy.bound_user_id()
    if uid is None:
        return None
    with _app_settings_cache_lock:
        cache = _app_settings_cache.get(uid)
        if cache is None:
            return None
        return cache.get(key)


def get_runtime_setting(key: str, env_fallback: str = "") -> str:
    """DB setting takes precedence; env_fallback (typically os.getenv) is the fallback."""
    val = get_cached_setting(key)
    if val is not None:
        return val
    return env_fallback


def delete_setting(conn: sqlite3.Connection, key: str) -> None:
    uid = tenancy.current_user_id()
    with _app_settings_cache_lock:
        cache = _app_settings_cache.get(uid)
        if cache is not None:
            cache.pop(key, None)
    conn.execute("DELETE FROM app_settings WHERE key = ?", (key,))


_DISPLAY_PREF_KEYS = frozenset(
    {
        "show_lead_image_in_article",
        "show_lead_image_as_thumb",
        "show_image_caption",
        "hide_shorts",
        "hide_unpremiered",
        "hide_members_only",
        "hide_paywalled",
        "auto_read_days",
        "hide_locked_comics",
        "inject_source_images",
        "katex_dollar_math",
        "fetch_full_content",
        "capture_page_topics",
    }
)
# Pre-built UPDATE statements (one per column) so conn.execute() never receives an f-string.
_DISPLAY_PREF_COLS: dict[str, str] = {k: k for k in _DISPLAY_PREF_KEYS}
_DISPLAY_PREF_SQLS: dict[str, str] = {k: f"UPDATE feed_display_prefs SET {k} = ? WHERE feed_url = ?" for k in _DISPLAY_PREF_KEYS}
_DISPLAY_PREF_DEFAULTS: dict = {
    "show_lead_image_in_article": 1,
    "show_lead_image_as_thumb": 1,
    "show_image_caption": -1,
    "hide_shorts": 0,
    "hide_unpremiered": 0,
    "hide_members_only": 0,
    "hide_paywalled": 0,
    "auto_read_days": 0,
    "hide_locked_comics": 0,
    "inject_source_images": 0,
    "katex_dollar_math": 0,
    "fetch_full_content": full_content_fetch.FEED_INHERIT,
    "capture_page_topics": full_content_fetch.FEED_INHERIT,
    "feed_thumbnail_url": None,
    "thumb_crop": "cover",
    "thumb_strategy": None,
    "smart_min_scale": None,
    "fill_zoom": None,
}
_VALID_THUMB_CROPS = frozenset(
    {
        "cover",
        "cover-top-left",
        "cover-top",
        "cover-top-right",
        "cover-left",
        "cover-right",
        "cover-bottom-left",
        "cover-bottom",
        "cover-bottom-right",
        "contain",
        "smart",
    }
)
_VALID_THUMB_STRATEGIES = frozenset({"inline", "media_rss"})

# Caption text that is purely a date (e.g. "June 12, 2026" or "6/12/2026") — strip it.
_DATE_ONLY_CAP_RE = re.compile(
    r"^\s*(?:\w+ \d{1,2},?\s*\d{4}|\d{1,2}[/.\-]\d{1,2}[/.\-]\d{2,4}|\d{4}[/.\-]\d{1,2}[/.\-]\d{1,2})\s*$",
    re.IGNORECASE,
)


def get_feed_display_prefs(conn: sqlite3.Connection, feed_url: str) -> dict:
    row = conn.execute("SELECT * FROM feed_display_prefs WHERE feed_url = ?", (feed_url,)).fetchone()
    if row:
        return dict(row)
    return {"feed_url": feed_url, **_DISPLAY_PREF_DEFAULTS}


def get_all_feed_display_prefs(conn: sqlite3.Connection) -> dict[str, dict]:
    rows = conn.execute("SELECT * FROM feed_display_prefs").fetchall()
    return {row["feed_url"]: dict(row) for row in rows}


def upsert_feed_display_pref(conn: sqlite3.Connection, feed_url: str, key: str, value: int) -> None:
    col = _DISPLAY_PREF_COLS.get(key)
    if col is None:
        raise ValueError(f"Unknown display pref key: {key}")
    conn.execute(
        "INSERT INTO feed_display_prefs (feed_url) VALUES (?) ON CONFLICT(feed_url) DO NOTHING",
        (feed_url,),
    )
    conn.execute(_DISPLAY_PREF_SQLS[col], (value, feed_url))


def upsert_feed_thumbnail_url(conn: sqlite3.Connection, feed_url: str, thumbnail_url: str | None) -> None:
    conn.execute(
        "INSERT INTO feed_display_prefs (feed_url) VALUES (?) ON CONFLICT(feed_url) DO NOTHING",
        (feed_url,),
    )
    conn.execute(
        "UPDATE feed_display_prefs SET feed_thumbnail_url = ? WHERE feed_url = ?",
        (thumbnail_url or None, feed_url),
    )


def upsert_feed_smart_min_scale(conn: sqlite3.Connection, feed_url: str, min_scale: float | None) -> None:
    value = min(1.0, max(0.5, min_scale)) if min_scale is not None else None
    conn.execute(
        "INSERT INTO feed_display_prefs (feed_url) VALUES (?) ON CONFLICT(feed_url) DO NOTHING",
        (feed_url,),
    )
    conn.execute(
        "UPDATE feed_display_prefs SET smart_min_scale = ? WHERE feed_url = ?",
        (value, feed_url),
    )


def upsert_feed_fill_zoom(conn: sqlite3.Connection, feed_url: str, zoom: float | None) -> None:
    value = min(2.0, max(0.5, zoom)) if zoom is not None else None
    conn.execute(
        "INSERT INTO feed_display_prefs (feed_url) VALUES (?) ON CONFLICT(feed_url) DO NOTHING",
        (feed_url,),
    )
    conn.execute(
        "UPDATE feed_display_prefs SET fill_zoom = ? WHERE feed_url = ?",
        (value, feed_url),
    )


def upsert_feed_thumb_strategy(conn: sqlite3.Connection, feed_url: str, strategy: str | None) -> None:
    s = strategy if strategy in _VALID_THUMB_STRATEGIES else None
    conn.execute(
        "INSERT INTO feed_display_prefs (feed_url) VALUES (?) ON CONFLICT(feed_url) DO NOTHING",
        (feed_url,),
    )
    conn.execute(
        "UPDATE feed_display_prefs SET thumb_strategy = ? WHERE feed_url = ?",
        (s, feed_url),
    )


_HIGHLIGHT_VALID_COLORS = frozenset({"yellow", "green", "blue", "pink", "orange"})
_HIGHLIGHT_VALID_SCOPES = frozenset({"global", "folder", "feed", "feeds", "folders"})


_HIGHLIGHT_VALID_TYPES = {
    "highlight",
    "mark_as_read",
    "email_article",
    "deduplicate",
    "webhook",
    "youtube_playlist",
    "instapaper",
    "quire",
    "tag_filter",
    "save_article",
}
_HIGHLIGHT_VALID_SEARCH_IN = {"title", "body", "both"}
_HIGHLIGHT_VALID_DELIVERY = {"immediately", "batch"}
_DEDUP_VALID_MATCH_METHODS = {"slug", "title", "both", "fuzzy", "safe"}
_DEDUP_FUZZY_PCT_DEFAULT = 80

# Dedup matching lives in services/dedup.py; these names are re-exported because route modules, automation_rules and tests import
# them from here.
from services.dedup import (  # noqa: E402
    _DEDUP_MIN_TITLE_WORDS,
    _safe_dedup_entry_slug,
    entry_url_slug,
    normalize_entry_title_for_dedupe,
)
from services.dedup import _cleanup_intra_feed_slug_dupes as _cleanup_intra_feed_dupes  # noqa: E402


def _clamp_min_title_words(words: int | None) -> int:
    """Title-length floor, clamped. Under 3 words a title is a category label."""
    try:
        v = int(words if words is not None else _DEDUP_MIN_TITLE_WORDS)
    except TypeError, ValueError:
        v = _DEDUP_MIN_TITLE_WORDS
    return max(3, min(10, v))


def _clamp_fuzzy_pct(pct: int | None) -> int:
    """Percent knob, clamped. Below 50% a fuzzy title match catches near-anything."""
    try:
        v = int(pct if pct is not None else _DEDUP_FUZZY_PCT_DEFAULT)
    except TypeError, ValueError:
        v = _DEDUP_FUZZY_PCT_DEFAULT
    return max(50, min(100, v))


def _dedup_fuzzy_threshold(pct: int | None) -> float:
    """Percent knob -> similarity ratio."""
    return _clamp_fuzzy_pct(pct) / 100.0


def get_highlight_keywords(conn: sqlite3.Connection) -> list[dict]:
    rows = conn.execute(
        "SELECT scope, scope_id, keyword, color, is_regex, enabled, type, search_in, delivery,"
        " email_to, batch_time, batch_count, cc_me, dedup_window_hours, dedup_fuzzy_pct,"
        " dedup_min_title_words,"
        " exclude_scope_ids, sort_order,"
        " webhook_url, webhook_format, webhook_batch,"
        " yt_playlist_id, yt_playlist_title, yt_include_shorts, yt_mark_read,"
        " yt_min_minutes, yt_max_minutes, rule_uid, label"
        " FROM highlight_keywords ORDER BY sort_order ASC, rowid ASC"
    ).fetchall()
    return [dict(r) for r in rows]


# Types whose `keyword` column is a plain search term (comma-list = OR of
# terms, already how a single rule's multi-keyword list works) rather than
# something else entirely -- deduplicate's is a match-method enum,
# youtube_playlist/instapaper/quire/save_article's is optional and its
# semantics for the merge/action types aren't confirmed. Scoped to exactly
# the two types the 2026-08-19 measurement found real mergeable groups in --
# see Plan.md "Utilities: find rules that could be one rule". tag_filter is
# mergeable too, but by spec union (_union_tag_filter_specs), not a keyword join.
_MERGEABLE_RULE_TYPES = frozenset({"highlight", "mark_as_read", "tag_filter"})

# Fields (besides scope/scope_id/type/search_in/is_regex/keyword) that change
# a rule's *behavior*, not just its keyword list. Two rules that differ here
# are NOT silently collapsed into one — see find_mergeable_rule_groups.
_MERGE_IDENTITY_FIELDS = ("color", "delivery", "email_to", "batch_time", "batch_count", "cc_me")


def find_mergeable_rule_groups(conn: sqlite3.Connection) -> tuple[list[dict], list[dict]]:
    """Groups of 2+ rules that share type/scope/scope_id/search_in/is_regex
    AND every other behavior-affecting field but differ only in keyword --
    collapsible into one rule with a joined keyword list. A keyword was one
    term until comma lists landed, so these accumulated from a time when one
    rule per term was the only way to add a second.

    Two rules can share type/scope/search_in/is_regex and still carry a
    *different* color/delivery/email setting and mean different things, so
    each identity key is further split into settings-consistent subgroups
    (2026-08-31) -- a folder with 2 orange rules, 2 blue rules and 1 green
    rule offers the orange pair and the blue pair as two independent
    mergeable groups, rather than reporting the whole five as one
    "these differ, edit to match" blob the way a single identity-wide check
    used to. The lone green rule has no partner either way and is silently
    excluded, same as a genuinely solo identity group always has been.
    ``mismatched`` now means the narrower thing that name always implied: 2+
    *leftover* singletons under the same identity that still disagree with
    each other once every already-agreeing pair has been pulled out.
    """
    rows = conn.execute(
        "SELECT scope, scope_id, keyword, color, is_regex, enabled, type, search_in, delivery,"
        " email_to, batch_time, batch_count, cc_me, sort_order"
        " FROM highlight_keywords WHERE type IN ('highlight', 'mark_as_read')"
        # Disabled tag filters are chip-tuning drafts (see toggle_feed_tag_filter); merging one would arm it.
        " OR (type = 'tag_filter' AND enabled = 1)"
        " ORDER BY sort_order ASC, rowid ASC"
    ).fetchall()
    by_identity: dict[tuple, list[dict]] = {}
    for r in rows:
        key = (r["type"], r["scope"], r["scope_id"], r["search_in"], bool(r["is_regex"]))
        by_identity.setdefault(key, []).append(dict(r))

    groups = []
    mismatched = []
    for key, group_rows in by_identity.items():
        if len(group_rows) < 2:
            continue
        rule_type, scope, scope_id, search_in, is_regex = key
        by_settings: dict[tuple, list[dict]] = {}
        for r in group_rows:
            by_settings.setdefault(tuple(r[f] for f in _MERGE_IDENTITY_FIELDS), []).append(r)

        leftover: list[dict] = []
        for settings_rows in by_settings.values():
            if len(settings_rows) >= 2:
                group = {
                    "type": rule_type,
                    "scope": scope,
                    "scope_id": scope_id,
                    "search_in": search_in,
                    "is_regex": is_regex,
                    "rules": settings_rows,
                }
                if rule_type == "tag_filter":
                    group["behavior_change"] = _tag_filter_merge_changes_behavior([r["keyword"] for r in settings_rows])
                groups.append(group)
            else:
                leftover.extend(settings_rows)
        if len(leftover) >= 2:
            mismatched.append(
                {
                    "type": rule_type,
                    "scope": scope,
                    "scope_id": scope_id,
                    "search_in": search_in,
                    "is_regex": is_regex,
                    "rules": leftover,
                }
            )
    return groups, mismatched


def merge_highlight_rule_group(
    conn: sqlite3.Connection,
    rule_type: str,
    scope: str,
    scope_id: str,
    search_in: str,
    is_regex: bool,
    color: str,
    delivery: str,
    email_to: str,
    batch_time: str,
    batch_count: int,
    cc_me: bool,
) -> dict | None:
    """Collapse every rule sharing (type, scope, scope_id, search_in, is_regex)
    AND the given color/delivery/email/batch/cc_me settings into one, joining
    their keywords (comma list, or regex alternation when is_regex).
    Re-derives the current matching rows from these criteria rather than
    trusting a caller-supplied row list, so a stale preview can't merge the
    wrong rows.

    The settings are part of the match itself now (2026-08-31), not a
    same-group precondition checked after the fact -- an identity key can
    hold several settings-distinct subgroups (see find_mergeable_rule_groups),
    and a merge must act on exactly the subgroup that was offered, not
    whichever settings happen to be most common among rows sharing identity.

    Refuses (returns None) when fewer than 2 rows match.
    """
    if rule_type not in _MERGEABLE_RULE_TYPES:
        return None
    # Same draft exclusion as find_mergeable_rule_groups: a disabled tag filter is never swept into a merge.
    enabled_clause = " AND enabled = 1" if rule_type == "tag_filter" else ""
    rows = conn.execute(
        "SELECT rowid, * FROM highlight_keywords"
        " WHERE type = ? AND scope = ? AND scope_id = ? AND search_in = ? AND is_regex = ?"
        " AND color = ? AND delivery = ? AND email_to = ? AND batch_time = ? AND batch_count = ? AND cc_me = ?"
        + enabled_clause
        + " ORDER BY sort_order ASC, rowid ASC",
        (
            rule_type,
            scope,
            scope_id,
            search_in,
            1 if is_regex else 0,
            color,
            delivery,
            email_to,
            batch_time,
            batch_count,
            1 if cc_me else 0,
        ),
    ).fetchall()
    if len(rows) < 2:
        return None
    rows = [dict(r) for r in rows]

    template = rows[0]
    seen_kw: list[str] = []
    for r in rows:
        kw = str(r["keyword"] or "").strip()
        if kw and kw not in seen_kw:
            seen_kw.append(kw)
    if rule_type == "tag_filter":
        merged_keyword = _union_tag_filter_specs(seen_kw)
    elif is_regex:
        merged_keyword = "|".join(f"({kw})" for kw in seen_kw)
    else:
        # Flatten to the individual-keyword level too, so merging a rule that
        # is already itself a comma list doesn't produce a repeated term.
        flat: list[str] = []
        seen_lower: set[str] = set()
        for kw in seen_kw:
            for part in kw.split(","):
                part = part.strip()
                if part and part.lower() not in seen_lower:
                    seen_lower.add(part.lower())
                    flat.append(part)
        merged_keyword = ", ".join(flat)

    min_sort_order = min(int(r["sort_order"] or 0) for r in rows)
    conn.execute(
        "DELETE FROM highlight_keywords"
        " WHERE type = ? AND scope = ? AND scope_id = ? AND search_in = ? AND is_regex = ?"
        " AND color = ? AND delivery = ? AND email_to = ? AND batch_time = ? AND batch_count = ? AND cc_me = ?" + enabled_clause,
        (
            rule_type,
            scope,
            scope_id,
            search_in,
            1 if is_regex else 0,
            color,
            delivery,
            email_to,
            batch_time,
            batch_count,
            1 if cc_me else 0,
        ),
    )
    add_highlight_keyword(
        conn,
        scope,
        scope_id,
        merged_keyword,
        template["color"],
        is_regex,
        rule_type,
        search_in,
        template["delivery"],
        template["email_to"],
        template["batch_time"],
        template["batch_count"],
        bool(template["cc_me"]),
        template["enabled"],
        template["dedup_window_hours"],
        template["exclude_scope_ids"],
        template["dedup_fuzzy_pct"],
        template["dedup_min_title_words"],
        template["webhook_url"],
        template["webhook_format"],
        bool(template["webhook_batch"]),
        template["yt_playlist_id"],
        template["yt_playlist_title"],
        bool(template["yt_include_shorts"]),
        bool(template["yt_mark_read"]),
        template["yt_min_minutes"],
        template["yt_max_minutes"],
        label=str(template.get("label") or ""),
    )
    conn.execute(
        "UPDATE highlight_keywords SET sort_order = ?"
        " WHERE type = ? AND scope = ? AND scope_id = ? AND search_in = ? AND is_regex = ? AND keyword = ?",
        (min_sort_order, rule_type, scope, scope_id, search_in, 1 if is_regex else 0, merged_keyword),
    )
    return {
        "type": rule_type,
        "scope": scope,
        "scope_id": scope_id,
        "search_in": search_in,
        "is_regex": is_regex,
        "keyword": merged_keyword,
        "merged_count": len(rows),
    }


def find_regex_convertible_rule_groups(conn: sqlite3.Connection) -> list[dict]:
    """Groups of 2+ rules sharing (type, scope, scope_id, search_in) that span
    BOTH a regex and a plain-text rule -- a same-is_regex group is already
    handled by find_mergeable_rule_groups, so this only surfaces the boundary
    case that one can't see. A plain-text keyword is always representable as
    an escaped regex (re.escape), so nothing here is actually blocked by
    is_regex disagreeing -- only by shown deliberately as its own suggestion
    tier since it changes a rule's is_regex, not just its keyword. Raised
    2026-08-31: "Lowe's" (plain) sat next to a regex rule on the same folder
    with nothing offering to combine them.

    Same behavior-identity guard as find_mergeable_rule_groups -- a group
    whose rules carry different color/delivery/email settings is excluded
    entirely, not silently merged with one side's settings winning."""
    rows = conn.execute(
        "SELECT scope, scope_id, keyword, color, is_regex, enabled, type, search_in, delivery,"
        " email_to, batch_time, batch_count, cc_me, sort_order"
        " FROM highlight_keywords WHERE type IN ('highlight', 'mark_as_read')"
        # Disabled tag filters are chip-tuning drafts (see toggle_feed_tag_filter); merging one would arm it.
        " OR (type = 'tag_filter' AND enabled = 1)"
        " ORDER BY sort_order ASC, rowid ASC"
    ).fetchall()
    by_identity: dict[tuple, list[dict]] = {}
    for r in rows:
        key = (r["type"], r["scope"], r["scope_id"], r["search_in"])
        by_identity.setdefault(key, []).append(dict(r))

    groups = []
    for key, group_rows in by_identity.items():
        if len(group_rows) < 2:
            continue
        if len({bool(r["is_regex"]) for r in group_rows}) < 2:
            continue  # all-regex or all-plain -- find_mergeable_rule_groups' territory
        if len({tuple(r[f] for f in _MERGE_IDENTITY_FIELDS) for r in group_rows}) > 1:
            continue
        rule_type, scope, scope_id, search_in = key
        groups.append(
            {
                "type": rule_type,
                "scope": scope,
                "scope_id": scope_id,
                "search_in": search_in,
                "rules": group_rows,
            }
        )
    return groups


def merge_regex_convertible_rule_group(
    conn: sqlite3.Connection,
    rule_type: str,
    scope: str,
    scope_id: str,
    search_in: str,
) -> dict | None:
    """Like merge_highlight_rule_group, but across the is_regex boundary: a
    plain-text keyword is escaped (re.escape) before joining, so the result
    matches exactly the same literal text the plain rule did -- see
    find_regex_convertible_rule_groups. Re-derives the current matching rows
    rather than trusting a caller-supplied list, same reasoning as the
    regex-only merge. Refuses (returns None) when fewer than 2 rows match, the
    group turns out to be single-is_regex after all (stale suggestion), or
    they carry different color/delivery/email settings."""
    if rule_type not in _MERGEABLE_RULE_TYPES:
        return None
    rows = conn.execute(
        "SELECT rowid, * FROM highlight_keywords"
        " WHERE type = ? AND scope = ? AND scope_id = ? AND search_in = ?"
        " ORDER BY sort_order ASC, rowid ASC",
        (rule_type, scope, scope_id, search_in),
    ).fetchall()
    if len(rows) < 2:
        return None
    rows = [dict(r) for r in rows]
    if len({bool(r["is_regex"]) for r in rows}) < 2:
        return None
    if len({tuple(r[f] for f in _MERGE_IDENTITY_FIELDS) for r in rows}) > 1:
        return None

    template = rows[0]
    seen_kw: list[str] = []
    for r in rows:
        kw = str(r["keyword"] or "").strip()
        if not kw:
            continue
        # A plain keyword's commas are OR-term separators (split_keyword_terms),
        # not literal text -- re.escape(kw) on the whole string turned "Pixel
        # Watch, Ryobi, Google Pixel" into one dead literal requiring all three
        # names adjacent with commas, instead of three alternatives. Escape each
        # term individually, then alternate them, before this whole row becomes
        # one outer (...) group below.
        pattern = kw if r["is_regex"] else "|".join(re.escape(term) for term in split_keyword_terms(kw))
        if pattern not in seen_kw:
            seen_kw.append(pattern)
    merged_keyword = "|".join(f"({kw})" for kw in seen_kw)

    min_sort_order = min(int(r["sort_order"] or 0) for r in rows)
    conn.execute(
        "DELETE FROM highlight_keywords WHERE type = ? AND scope = ? AND scope_id = ? AND search_in = ?",
        (rule_type, scope, scope_id, search_in),
    )
    add_highlight_keyword(
        conn,
        scope,
        scope_id,
        merged_keyword,
        template["color"],
        True,
        rule_type,
        search_in,
        template["delivery"],
        template["email_to"],
        template["batch_time"],
        template["batch_count"],
        bool(template["cc_me"]),
        template["enabled"],
        template["dedup_window_hours"],
        template["exclude_scope_ids"],
        template["dedup_fuzzy_pct"],
        template["dedup_min_title_words"],
        template["webhook_url"],
        template["webhook_format"],
        bool(template["webhook_batch"]),
        template["yt_playlist_id"],
        template["yt_playlist_title"],
        bool(template["yt_include_shorts"]),
        bool(template["yt_mark_read"]),
        template["yt_min_minutes"],
        template["yt_max_minutes"],
        label=str(template.get("label") or ""),
    )
    conn.execute(
        "UPDATE highlight_keywords SET sort_order = ?"
        " WHERE type = ? AND scope = ? AND scope_id = ? AND search_in = ? AND is_regex = 1 AND keyword = ?",
        (min_sort_order, rule_type, scope, scope_id, search_in, merged_keyword),
    )
    return {
        "type": rule_type,
        "scope": scope,
        "scope_id": scope_id,
        "search_in": search_in,
        "is_regex": True,
        "keyword": merged_keyword,
        "merged_count": len(rows),
    }


def _split_plain_keywords(raw: str) -> set[str]:
    return {kw.strip().lower() for kw in (raw or "").split(",") if kw.strip()}


def find_redundant_feed_rules(conn: sqlite3.Connection) -> list[dict]:
    """Plain keyword feed rules covered by a folder rule, plus tag_filter feed rules that do nothing (see the two helpers)."""
    return _find_redundant_keyword_feed_rules(conn) + _find_redundant_tag_filter_feed_rules(conn)


def _find_redundant_tag_filter_feed_rules(conn: sqlite3.Connection) -> list[dict]:
    """Enabled tag_filter feed rules that change nothing: ``no_op`` when the spec has no ``-``/``++`` tag at all (a ``+`` only
    rescues from its own rule's drops, never another rule's), ``covered`` when every ``-`` tag is already dropped by an enabled
    folder tag_filter rule on a folder holding the feed that rescues no more than the feed rule does. ``++`` feed rules are never
    flagged: a whitelist's cut isn't reproduced by anything else."""
    feed_rows = conn.execute(
        "SELECT scope_id AS feed_url, keyword FROM highlight_keywords WHERE type = 'tag_filter' AND scope = 'feed' AND enabled = 1"
    ).fetchall()
    if not feed_rows:
        return []
    folder_specs: dict[int, list[tuple[set[str], set[str]]]] = {}  # folder_id -> [(exclude, saving)]
    for r in conn.execute(
        "SELECT scope_id, keyword FROM highlight_keywords WHERE type = 'tag_filter' AND scope = 'folder' AND enabled = 1"
    ).fetchall():
        try:
            fid = int(r["scope_id"])
        except TypeError, ValueError:
            continue
        req, good, exc = parse_tag_filter_spec(str(r["keyword"] or ""))
        folder_specs.setdefault(fid, []).append((exc, req | good))

    redundant = []
    for r in feed_rows:
        feed_url = str(r["feed_url"])
        req, good, exc = parse_tag_filter_spec(str(r["keyword"] or ""))
        if req:
            continue
        base = {"feed_url": feed_url, "keyword": r["keyword"], "type": "tag_filter", "search_in": "title"}
        if not exc:
            redundant.append({**base, "reason": "no_op", "covering_folder_id": None})
            continue
        feed_saving = good
        for (fid,) in conn.execute("SELECT folder_id FROM folder_feeds WHERE feed_url = ?", (feed_url,)).fetchall():
            covered = set()
            for f_exc, f_saving in folder_specs.get(int(fid), []):
                if f_saving <= feed_saving:
                    covered |= f_exc
            if exc <= covered:
                redundant.append({**base, "reason": "covered", "covering_folder_id": int(fid)})
                break
    return redundant


def _find_redundant_keyword_feed_rules(conn: sqlite3.Connection) -> list[dict]:
    """Feed-scoped rules already fully covered by a same-type, same-search_in
    folder rule on a folder the feed belongs to -- the folder rule already
    catches everything the feed rule would, so the feed rule does nothing.

    Plain (non-regex) rules only: whether one regex's language is a subset of
    another's isn't decidable this way, so a regex rule is never flagged.
    """
    feed_rows = conn.execute(
        "SELECT scope_id AS feed_url, keyword, type, search_in FROM highlight_keywords"
        " WHERE scope = 'feed' AND is_regex = 0 AND type IN ('highlight', 'mark_as_read')"
    ).fetchall()
    if not feed_rows:
        return []
    folder_rows = conn.execute(
        "SELECT scope_id AS folder_id, keyword, type, search_in FROM highlight_keywords"
        " WHERE scope = 'folder' AND is_regex = 0 AND type IN ('highlight', 'mark_as_read')"
    ).fetchall()
    if not folder_rows:
        return []
    folder_keywords: dict[tuple, set[str]] = {}
    for r in folder_rows:
        key = (r["type"], r["search_in"], str(r["folder_id"]))
        folder_keywords.setdefault(key, set()).update(_split_plain_keywords(r["keyword"]))

    feed_urls = {str(r["feed_url"]) for r in feed_rows}
    placeholders = ",".join("?" for _ in feed_urls)
    folder_membership = conn.execute(
        f"SELECT feed_url, folder_id FROM folder_feeds WHERE feed_url IN ({placeholders})",
        list(feed_urls),
    ).fetchall()
    feed_folder_ids: dict[str, set[int]] = {}
    for r in folder_membership:
        feed_folder_ids.setdefault(str(r["feed_url"]), set()).add(int(r["folder_id"]))

    redundant = []
    for r in feed_rows:
        feed_url = str(r["feed_url"])
        feed_kws = _split_plain_keywords(r["keyword"])
        if not feed_kws:
            continue
        for folder_id in feed_folder_ids.get(feed_url, set()):
            covering = folder_keywords.get((r["type"], r["search_in"], str(folder_id)))
            if covering and feed_kws.issubset(covering):
                redundant.append(
                    {
                        "feed_url": feed_url,
                        "keyword": r["keyword"],
                        "type": r["type"],
                        "search_in": r["search_in"],
                        "covering_folder_id": folder_id,
                    }
                )
                break
    return redundant


# On a cross-rule sign conflict the union keeps the strongest: a drop outranks a whitelist, which outranks a rescue.
_TAG_FILTER_SIGN_RANK = {"-": 2, "++": 1, "+": 0}


def _union_tag_filter_specs(specs: list[str]) -> str:
    """One tag_filter spec holding every signed tag in *specs*, first-seen order, deduped by tag."""
    by_tag: dict[str, str] = {}
    for spec in specs:
        for token in spec.split(","):
            token = token.strip()
            sign = next((p for p in ("++", "+", "-") if token.startswith(p)), "")
            tag = normalize_tag_value(token[len(sign) :])
            sign = sign or "+"  # bare = good, as in parse_tag_filter_spec
            if tag and (tag not in by_tag or _TAG_FILTER_SIGN_RANK[sign] > _TAG_FILTER_SIGN_RANK[by_tag[tag]]):
                by_tag[tag] = sign
    return ", ".join(f"{sign}{tag}" for tag, sign in by_tag.items())


def _tag_filter_merge_changes_behavior(specs: list[str]) -> bool:
    """Whether merging these tag_filter rules into one changes what they mark read.

    A ``+``/``++`` tag only acts on its own rule's ``-`` tags, so separate rules are exactly their union only when they all share the
    same ``+`` and ``++`` sets (e.g. all pure ``-`` rules). Otherwise a merge lets one rule's rescue reach another's drops."""
    parsed = [parse_tag_filter_spec(s) for s in specs]
    return any((req, good) != (parsed[0][0], parsed[0][1]) for req, good, _exc in parsed[1:])


def _merge_tag_filter_specs(conn: sqlite3.Connection, feed_url: str, incoming: str) -> str | None:
    """Fold *incoming* into the feed's existing tag_filter spec, and drop the
    now-redundant rows. Returns the merged spec, or None when there is nothing to
    merge with.

    Signed tokens are deduped by TAG, with the incoming sign winning: re-adding
    ``+rust`` where ``-rust`` was set is a correction, not a contradiction to
    preserve.
    """
    rows = conn.execute(
        "SELECT rowid, keyword FROM highlight_keywords WHERE type = 'tag_filter' AND scope = 'feed' AND scope_id = ? ORDER BY rowid",
        (feed_url,),
    ).fetchall()
    if not rows:
        return None

    by_tag: dict[str, str] = {}  # tag -> sign, insertion-ordered
    for source in [str(r["keyword"] or "") for r in rows] + [incoming]:
        for token in source.split(","):
            token = token.strip()
            if not token:
                continue
            sign, tag = "", token
            for prefix in ("++", "+", "-"):
                if token.startswith(prefix):
                    sign, tag = prefix, token[len(prefix) :]
                    break
            tag = normalize_tag_value(tag)
            if tag:
                by_tag[tag] = sign
    # Delete ALL of them, not just the extras: the caller's INSERT OR REPLACE keys
    # on the keyword, which the merge has just changed, so leaving the first row
    # behind would leave a rival rather than replace it.
    conn.executemany(
        "DELETE FROM highlight_keywords WHERE rowid = ?",
        [(r["rowid"],) for r in rows],
    )
    if len(rows) > 1:
        LOGGER.info("[tag-filter] merged %d duplicate rule(s) for %s", len(rows) - 1, feed_url)
    return ", ".join(f"{sign}{tag}" for tag, sign in by_tag.items())


def add_highlight_keyword(
    conn: sqlite3.Connection,
    scope: str,
    scope_id: str,
    keyword: str,
    color: str,
    is_regex: bool = False,
    rule_type: str = "highlight",
    search_in: str = "title",
    delivery: str = "immediately",
    email_to: str = "",
    batch_time: str = "",
    batch_count: int = 0,
    cc_me: bool = False,
    enabled: int = 0,
    dedup_window_hours: int = 168,
    exclude_scope_ids: str = "",
    dedup_fuzzy_pct: int = _DEDUP_FUZZY_PCT_DEFAULT,
    dedup_min_title_words: int = _DEDUP_MIN_TITLE_WORDS,
    webhook_url: str = "",
    webhook_format: str = "generic",
    webhook_batch: bool = False,
    yt_playlist_id: str = "",
    yt_playlist_title: str = "",
    yt_include_shorts: bool = False,
    yt_mark_read: bool = True,
    yt_min_minutes: int = 0,
    yt_max_minutes: int = 0,
    rule_uid: str = "",
    label: str = "",
) -> None:
    if scope not in _HIGHLIGHT_VALID_SCOPES:
        raise ValueError(f"Invalid scope: {scope}")
    if color not in _HIGHLIGHT_VALID_COLORS:
        color = "yellow"
    if rule_type not in _HIGHLIGHT_VALID_TYPES:
        rule_type = "highlight"
    if search_in not in _HIGHLIGHT_VALID_SEARCH_IN:
        search_in = "title"
    if delivery not in _HIGHLIGHT_VALID_DELIVERY:
        delivery = "immediately"
    if webhook_format not in WEBHOOK_VALID_FORMATS:
        webhook_format = "generic"
    # A feed can only have ONE tag_filter rule: get_feed_tag_filter_rule fetches a
    # single row, so a second one is edited by nothing while still executing — two
    # rules on the same dev.to feed ("-rust, -powerbi" and "-rust, -hindi,
    # -javascript, -powerbi, -aws") were both running, invisibly. Merge into the
    # existing rule rather than adding a rival.
    if rule_type == "tag_filter" and scope == "feed":
        merged = _merge_tag_filter_specs(conn, scope_id, keyword)
        if merged is not None:
            keyword = merged
    conn.execute(
        "INSERT OR REPLACE INTO highlight_keywords"
        " (scope, scope_id, keyword, color, is_regex, enabled, type, search_in, delivery,"
        "  email_to, batch_time, batch_count, cc_me, dedup_window_hours, exclude_scope_ids,"
        "  dedup_fuzzy_pct, dedup_min_title_words,"
        "  webhook_url, webhook_format, webhook_batch,"
        "  yt_playlist_id, yt_playlist_title, yt_include_shorts, yt_mark_read,"
        "  yt_min_minutes, yt_max_minutes, rule_uid, label)"
        " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
        (
            scope,
            scope_id,
            keyword.strip(),
            color,
            1 if is_regex else 0,
            1 if enabled else 0,
            rule_type,
            search_in,
            delivery,
            email_to.strip(),
            batch_time.strip(),
            max(0, int(batch_count or 0)),
            1 if cc_me else 0,
            max(1, int(dedup_window_hours or 168)),
            exclude_scope_ids.strip(),
            _clamp_fuzzy_pct(dedup_fuzzy_pct),
            _clamp_min_title_words(dedup_min_title_words),
            webhook_url.strip(),
            webhook_format,
            1 if webhook_batch else 0,
            yt_playlist_id.strip(),
            yt_playlist_title.strip(),
            1 if yt_include_shorts else 0,
            1 if yt_mark_read else 0,
            max(0, int(yt_min_minutes or 0)),
            max(0, int(yt_max_minutes or 0)),
            rule_uid.strip() or secrets.token_hex(16),
            label.strip(),
        ),
    )


def get_email_contacts(conn: sqlite3.Connection) -> list[dict]:
    rows = conn.execute("SELECT id, label, address FROM email_contacts ORDER BY label").fetchall()
    return [dict(r) for r in rows]


def remove_highlight_keyword(conn: sqlite3.Connection, scope: str, scope_id: str, keyword: str) -> None:
    conn.execute(
        "DELETE FROM highlight_keywords WHERE scope = ? AND scope_id = ? AND keyword = ?",
        (scope, scope_id, keyword.strip()),
    )


_TRIVIAL_CAPTIONS = frozenset(
    {
        "responsive image",
        "image",
        "photo",
        "picture",
        "img",
        "thumbnail",
        "banner",
        "featured image",
        "header image",
    }
)
_FILENAME_CAPTION_RE = re.compile(
    r"(?:\.(jpe?g|png|gif|webp|avif|svg|bmp|tiff?)$|^(?:DSC|IMG|dsc|img)[_-]?\d{3,})",
    re.IGNORECASE,
)


def should_show_caption(caption: str | None, *, entry_title: str | None, content_html: str | None, pref: int) -> bool:
    """Return whether the image caption should be rendered.

    pref: -1 = auto-suppress heuristics, 0 = never, 1 = always.
    """
    if not caption:
        return False
    if pref == 0:
        return False
    if pref == 1:
        return True
    # Auto heuristics
    stripped = caption.strip()
    if stripped.lower() in _TRIVIAL_CAPTIONS:
        return False
    if _FILENAME_CAPTION_RE.search(stripped):
        return False
    if entry_title and stripped.lower() == entry_title.strip().lower():
        return False
    if content_html and stripped in content_html:
        # Cheap check above can false-positive on attribute text: a webcomic's own
        # <img alt="hovertext" title="hovertext"> makes the hovertext a raw substring
        # of content_html even though it is invisible without a hover — not "already
        # shown" the way a rendered <figcaption> is. Confirm against visible text only.
        from bs4 import BeautifulSoup

        if stripped in BeautifulSoup(content_html, "html.parser").get_text():
            return False
    return True


def normalize_auto_refresh_minutes(value: int) -> int:
    if value <= 0:
        return 0
    return max(value, MIN_AUTO_REFRESH_MINUTES)


def parse_epoch_setting(value: str | None) -> float | None:
    if value is None:
        return None
    try:
        parsed = float(value)
    except TypeError, ValueError:
        return None
    if parsed <= 0:
        return None
    return parsed


def get_auto_refresh_minutes(conn: sqlite3.Connection) -> int:
    raw_value = get_setting(conn, AUTO_REFRESH_SETTING_KEY)
    if raw_value is None:
        return get_instance_default_auto_refresh()
    try:
        return normalize_auto_refresh_minutes(int(raw_value))
    except ValueError:
        return get_instance_default_auto_refresh()


def purge_lower_level_folders(conn: sqlite3.Connection) -> None:
    root_id = get_root_folder_id(conn)
    rows = conn.execute(
        """
        WITH RECURSIVE tree(id, parent_id, depth, top_level_id) AS (
            SELECT id, parent_id, 0 AS depth, id AS top_level_id
            FROM folders
            WHERE id = ?
            UNION ALL
            SELECT f.id,
                   f.parent_id,
                   t.depth + 1,
                   CASE WHEN t.depth = 0 THEN f.id ELSE t.top_level_id END AS top_level_id
            FROM folders f
            JOIN tree t ON f.parent_id = t.id
        )
        SELECT id, depth, top_level_id
        FROM tree
        WHERE depth >= 2
        """,
        (root_id,),
    ).fetchall()

    nested_folder_ids: list[int] = []
    for row in rows:
        folder_id = int(row["id"])
        top_level_id = int(row["top_level_id"])
        nested_folder_ids.append(folder_id)
        conn.execute(
            """
            INSERT OR IGNORE INTO folder_feeds (folder_id, feed_url)
            SELECT ?, feed_url
            FROM folder_feeds
            WHERE folder_id = ?
            """,
            (top_level_id, folder_id),
        )

    if not nested_folder_ids:
        return

    placeholders = ",".join("?" for _ in nested_folder_ids)
    conn.execute(
        f"DELETE FROM folder_feeds WHERE folder_id IN ({placeholders})",
        nested_folder_ids,
    )
    conn.execute(
        f"DELETE FROM folders WHERE id IN ({placeholders})",
        nested_folder_ids,
    )


def get_root_folder_id(conn: sqlite3.Connection) -> int:
    row = conn.execute(
        "SELECT id FROM folders WHERE name = ? AND parent_id IS NULL",
        (ROOT_FOLDER_NAME,),
    ).fetchone()
    if not row:
        raise RuntimeError("Root folder is missing.")
    return int(row["id"])


def _get_or_create_folder_by_name(conn: sqlite3.Connection, name: str) -> int:
    """Return the id of a top-level folder with `name`, creating it under root if absent."""
    row = conn.execute("SELECT id FROM folders WHERE name = ? LIMIT 1", (name,)).fetchone()
    if row:
        return int(row["id"])
    root_id = get_root_folder_id(conn)
    conn.execute("INSERT INTO folders (name, parent_id) VALUES (?, ?)", (name, root_id))
    return int(conn.execute("SELECT last_insert_rowid()").fetchone()[0])


def _get_lectio_folder_id(conn: sqlite3.Connection) -> int | None:
    root_id = get_root_folder_id(conn)
    row = conn.execute(
        "SELECT id FROM folders WHERE name = ? AND parent_id = ?",
        (_LECTIO_FOLDER_NAME, root_id),
    ).fetchone()
    return int(row["id"]) if row else None


def get_descendant_folder_ids(conn: sqlite3.Connection, folder_id: int) -> list[int]:
    rows = conn.execute(
        """
        WITH RECURSIVE descendants(id) AS (
            SELECT id FROM folders WHERE id = ?
            UNION ALL
            SELECT f.id
            FROM folders f
            JOIN descendants d ON f.parent_id = d.id
        )
        SELECT id FROM descendants
        """,
        (folder_id,),
    ).fetchall()
    return [int(row["id"]) for row in rows]


def get_folder_rows(conn: sqlite3.Connection) -> list[sqlite3.Row]:
    debug_clause = "" if DEBUG_MODE else " AND f.name != :exclude"
    params: dict = {"root": ROOT_FOLDER_NAME}
    if not DEBUG_MODE:
        params["exclude"] = _LECTIO_FOLDER_NAME
    return conn.execute(
        f"""
        SELECT
            f.id,
            f.name,
            f.cadence_minutes,
            CASE WHEN f.parent_id IS NULL THEN 0 ELSE 1 END AS depth,
            CASE WHEN f.parent_id IS NULL THEN f.name ELSE root.name || ' / ' || f.name END AS path,
            (
                SELECT COUNT(*)
                FROM folder_feeds ff
                WHERE ff.folder_id = f.id
            ) AS feed_count
        FROM folders f
        LEFT JOIN folders root ON f.parent_id = root.id
        WHERE (f.parent_id IS NULL OR f.parent_id = (
            SELECT id FROM folders WHERE name = :root AND parent_id IS NULL
        )){debug_clause}
        ORDER BY path
        """,
        params,
    ).fetchall()


def get_folder_options(conn: sqlite3.Connection) -> list[FolderOption]:
    debug_clause = "" if DEBUG_MODE else " AND f.name != :exclude"
    params: dict = {"root": ROOT_FOLDER_NAME}
    if not DEBUG_MODE:
        params["exclude"] = _LECTIO_FOLDER_NAME
    rows = conn.execute(
        f"""
        SELECT
            f.id,
            f.name,
            CASE WHEN f.parent_id IS NULL THEN 0 ELSE 1 END AS depth,
            CASE WHEN f.parent_id IS NULL THEN f.name ELSE root.name || ' / ' || f.name END AS path
        FROM folders f
        LEFT JOIN folders root ON f.parent_id = root.id
        WHERE (f.parent_id IS NULL OR f.parent_id = (
            SELECT id FROM folders WHERE name = :root AND parent_id IS NULL
        )){debug_clause}
        ORDER BY path
        """,
        params,
    ).fetchall()
    return [FolderOption(id=int(r["id"]), name=r["name"], path=r["path"], depth=int(r["depth"])) for r in rows]


def get_direct_feed_urls_by_folder(conn: sqlite3.Connection) -> dict[int, list[str]]:
    rows = conn.execute("SELECT folder_id, feed_url FROM folder_feeds ORDER BY feed_url").fetchall()
    by_folder: dict[int, list[str]] = {}
    for row in rows:
        folder_id = int(row["folder_id"])
        by_folder.setdefault(folder_id, []).append(str(row["feed_url"]))
    return by_folder


def get_folder_feed_urls(conn: sqlite3.Connection, folder_id: int) -> set[str]:
    # The virtual "Uncategorized" folder has no folder_feeds rows; its members are
    # every reader feed not in any folder. Resolving it here lets all folder
    # actions (mark-read, refresh, …) operate on it uniformly. The Saved
    # Articles virtual feed is excluded here too — it must never be
    # actionable (mark-read/refresh/moved) as if it were an ordinary
    # unfoldered feed; the Saved sidebar's own reachability need is served
    # elsewhere (_home_inner's snapshot dict), not through this resolver.
    if folder_id == UNCATEGORIZED_FOLDER_ID:
        return get_all_reader_feed_urls() - get_all_feed_urls(conn) - {saved_articles_service.SAVED_FEED_URL}
    # "All Feeds" (root) covers every reader feed, including orphans that live in
    # no folder. The home view widens root the same way for display and counts, so
    # folder actions (mark-read/older, refresh) must match — otherwise entries in
    # uncategorized feeds show under All Feeds but can't be marked from there.
    # The Saved Articles virtual feed (lectio:saved) is excluded here too: unlike
    # Uncategorized (which keeps it reachable for the Saved sidebar's own
    # Uncategorized grouping — see the home route), root has no such need, and
    # it must never show as a subscription — or be actionable — in Feeds mode.
    if folder_id == get_root_folder_id(conn):
        return get_all_reader_feed_urls() - {saved_articles_service.SAVED_FEED_URL}
    descendant_ids = get_descendant_folder_ids(conn, folder_id)
    placeholders = ",".join("?" for _ in descendant_ids)
    rows = conn.execute(
        f"SELECT DISTINCT feed_url FROM folder_feeds WHERE folder_id IN ({placeholders})",
        descendant_ids,
    ).fetchall()
    return {str(r["feed_url"]) for r in rows}


def get_all_feed_urls(conn: sqlite3.Connection) -> set[str]:
    rows = conn.execute("SELECT DISTINCT feed_url FROM folder_feeds").fetchall()
    return {str(r["feed_url"]) for r in rows}


def get_kept_feed_urls(conn: sqlite3.Connection | None = None) -> set[str]:
    """Feeds unsubscribed-but-retained for their curation (see kept_feeds table).

    They still exist in reader (so their starred/tagged entries and tags survive)
    but are hidden from the feed tree, All Feeds, Uncategorized, and counts — only
    the Saved/Kept view surfaces them, grouped under their original feed name.
    """
    if conn is not None:
        rows = conn.execute("SELECT feed_url FROM kept_feeds").fetchall()
        return {str(r["feed_url"]) for r in rows}
    with get_meta_connection() as own:
        rows = own.execute("SELECT feed_url FROM kept_feeds").fetchall()
        return {str(r["feed_url"]) for r in rows}


def get_feeds_needing_replacement(conn: sqlite3.Connection | None = None) -> set[str]:
    """Feeds the user triaged as dead/needs-replacement (disabled + worklisted)."""
    if conn is not None:
        return {str(r["feed_url"]) for r in conn.execute("SELECT feed_url FROM feeds_needing_replacement")}
    with get_meta_connection() as own:
        return {str(r["feed_url"]) for r in own.execute("SELECT feed_url FROM feeds_needing_replacement")}


def unsubscribed_feed_urls_among(feed_urls: Iterable[str | None]) -> set[str]:
    """Which of these feeds you are no longer subscribed to.

    Two different states read as "unsubscribed" to a reader and both belong here:
    a **kept feed** still exists in reader but is hidden from the tree, and an
    **orphan archive**'s feed is gone from reader entirely. What they have in
    common is the only thing that matters at the point of use — there is no
    subscription behind the name, so clicking through to the feed will not show
    you more of it.

    Deliberately not "has no folder row": a feed in no folder is still
    subscribed, it just lives under the virtual Uncategorized folder, and marking
    those would be wrong.

    Scoped to the feeds actually on screen so the template carries a handful of
    URLs rather than the whole library.
    """
    wanted = {str(u) for u in feed_urls if u}
    if not wanted:
        return set()
    return wanted - get_all_reader_feed_urls(include_kept=False)


def get_all_reader_feed_urls(include_kept: bool = False) -> set[str]:
    """Return every feed URL the reader is subscribed to.

    Unlike get_all_feed_urls (which reads folder_feeds), this covers feeds that
    were never assigned to a folder — the source set for the virtual
    "Uncategorized" folder. Read straight from the reader DB to stay O(feeds)
    without materializing full feed objects.

    Kept feeds (unsubscribed-but-retained for curation) are excluded by default
    so they don't resurface in the tree / All Feeds / Uncategorized / counts. The
    Saved/Kept view passes include_kept=True to browse them.

    Goes through get_reader()'s pooled connection rather than a fresh
    sqlite3.connect() per call -- the same anti-pattern fixed 2026-09-03 for
    list_entries_for_feeds' sort paths (Plan.md Tier 1). This function is called
    unconditionally on every home-route render (nested inside the
    "structure_snapshot" perf tick, which is what was actually timing this, not
    the cached snapshot lookup its name suggests) plus a dozen other call
    sites, so a raw connection here paid file-open/schema-load cost, with a
    shorter busy_timeout (5s vs 10s) than the pooled connection, on every
    single request -- and was invisible to slow-SQL logging since a bare
    sqlite3.connect() bypasses _TimedConnection entirely.

    Logged with its own progress_steps when slow, same reasoning as
    get_tagged_entry_keys: the `{... for r in db.execute(...)}` comprehension
    fetches lazily, so a plain execute()-level timer would only ever see the
    prepare cost, not the real scan -- confirmed live 2026-09-03 when this
    function measured 3166ms with nothing logged to explain it.
    """
    with get_reader() as reader:
        db = reader._storage.get_db()
        _start = time.perf_counter()
        urls = {str(r[0]) for r in db.execute("SELECT url FROM feeds")}
        _elapsed_ms = int((time.perf_counter() - _start) * 1000)
        if _elapsed_ms > 200:
            LOGGER.info(
                "[perf] get_all_reader_feed_urls=%dms rows=%d progress_steps=%d",
                _elapsed_ms,
                len(urls),
                getattr(db, "_lectio_progress_steps", 0),
            )
    if not include_kept:
        urls -= get_kept_feed_urls()
    return urls


# A "feeds" rule scope targets an explicit set of feeds (not a whole folder). Its
# scope_id is the feed URLs joined by newline (URLs may contain commas, so newline
# is the safe separator). "folders" is the same idea one level up: several whole
# folders in one rule instead of duplicating the rule per folder (requested by
# Josh 2026-08-31) — scope_id is the folder ids joined by newline. These helpers
# centralize parse + scope→feed-set + the per-feed "is this feed in scope" check
# used by the after-refresh runners.
_FEEDS_SCOPE_SEP = "\n"
_FOLDERS_SCOPE_SEP = "\n"


def parse_feeds_scope_id(scope_id: str) -> list[str]:
    return [u.strip() for u in (scope_id or "").split(_FEEDS_SCOPE_SEP) if u.strip()]


def parse_folders_scope_id(scope_id: str) -> list[int]:
    return [int(s) for s in (scope_id or "").split(_FOLDERS_SCOPE_SEP) if s.strip().isdigit()]


def rule_scope_folder_ids(scope: str, scope_id: str) -> set[int]:
    """Folder id(s) a rule's scope references, for "folder" and "folders" alike —
    the single choke point the after-refresh runners' folder-prewarm passes use so
    a multi-folder rule's folders are fetched same as a single-folder rule's."""
    if scope == "folder":
        return {int(scope_id)} if str(scope_id or "").isdigit() else set()
    if scope == "folders":
        return set(parse_folders_scope_id(scope_id))
    return set()


def rule_scope_folder_feed_set(scope: str, scope_id: str, folder_feed_map: dict[int, set[str]]) -> set[str] | None:
    """Prewarmed feed set for a folder-based rule scope (single folder or several),
    for feed_in_rule_scope's per-feed test. None for non-folder scopes."""
    ids = rule_scope_folder_ids(scope, scope_id)
    if not ids:
        return None
    feed_urls: set[str] = set()
    for fid in ids:
        feed_urls |= folder_feed_map.get(fid, set())
    return feed_urls


def resolve_rule_feed_urls(conn: sqlite3.Connection, scope: str, scope_id: str) -> set[str] | None:
    """Return the set of feed URLs a rule applies to, or None for global (all feeds)."""
    if scope == "global":
        return None
    if scope == "folder":
        try:
            return get_folder_feed_urls(conn, int(scope_id))
        except ValueError, TypeError:
            return set()
    if scope == "folders":
        feed_urls: set[str] = set()
        for fid in parse_folders_scope_id(scope_id):
            feed_urls |= get_folder_feed_urls(conn, fid)
        return feed_urls
    if scope == "feed":
        return {scope_id} if scope_id else set()
    if scope == "feeds":
        return set(parse_feeds_scope_id(scope_id))
    return set()


def feed_in_rule_scope(scope: str, scope_id: str, feed_url: str, folder_feed_urls: set[str] | None) -> bool:
    """Per-feed scope test for the after-refresh runners. ``folder_feed_urls`` is the
    prefetched feed set for a folder-scoped rule (ignored for other scopes)."""
    if scope == "global":
        return True
    if scope in ("folder", "folders"):
        return feed_url in (folder_feed_urls or set())
    if scope == "feed":
        return scope_id == feed_url
    if scope == "feeds":
        return feed_url in parse_feeds_scope_id(scope_id)
    return False


def get_disabled_feed_urls(conn: sqlite3.Connection) -> set[str]:
    rows = conn.execute("SELECT feed_url FROM disabled_feeds").fetchall()
    return {str(r["feed_url"]) for r in rows}


def get_feed_url_rewrites(feed_url: str) -> list[tuple[str, str]]:
    """[(from_host, to_host)] host-alias rules for *feed_url*, for the ingest
    rewrite hook. Opens its own connection: called from the sanitizing parser,
    which runs in the caller's tenancy context but has no meta connection."""
    try:
        conn = sqlite3.connect(str(tenancy.meta_db_path()), timeout=5.0)
        try:
            return [
                (str(f), str(t))
                for f, t in conn.execute(
                    "SELECT from_host, to_host FROM feed_url_rewrites WHERE feed_url = ?",
                    (feed_url,),
                )
            ]
        finally:
            conn.close()
    except Exception:  # noqa: BLE001 — a rewrite lookup must never break a parse
        return []


def get_dedupe_host_aliases() -> dict[str, str]:
    """{from_host: to_host} across every feed's URL-rewrite rules, for the saved
    dedupe scan.

    A feed_url_rewrites rule is the user declaring "this author moved domains" —
    ingest already stamps new entries with the current host, but articles saved
    before the move keep the dead host, so an old sadh.life/post/x and a new
    tush.ar/post/x are the same article the host-scoped dedupe key can't pair.
    Folding the migrated host onto its target makes them one key. Because the
    map is user-declared (not a heuristic), it can't false-match the way a
    cross-host slug tier would."""
    try:
        conn = sqlite3.connect(str(tenancy.meta_db_path()), timeout=5.0)
        try:
            aliases: dict[str, str] = {}
            for f, t in conn.execute("SELECT DISTINCT from_host, to_host FROM feed_url_rewrites"):
                fh = str(f).lower().removeprefix("www.")
                th = str(t).lower().removeprefix("www.")
                if fh and th and fh != th:
                    aliases[fh] = th
            # Follow chained migrations (a -> b, b -> c) to the final host.
            for src in list(aliases):
                seen = {src}
                dst = aliases[src]
                while dst in aliases and dst not in seen:
                    seen.add(dst)
                    dst = aliases[dst]
                aliases[src] = dst
            return aliases
        finally:
            conn.close()
    except Exception:  # noqa: BLE001 — dedupe must degrade to host-scoped, not break
        return {}


def get_browser_ua_feed_urls(conn: sqlite3.Connection) -> set[str]:
    """Feeds whose fetch should use a browser identity (UA + headers).

    A feed lands here only after an honest-UA fetch was *refused* (HTTP
    403/415/429/503 or a hang) — see `_maybe_flag_browser_ua`. The request hook in
    services.reader_api consults this set so the next fetch escalates. This is a
    good-citizen escalation (only on refusal, never preemptive); it does not evade
    IP-level blocks."""
    rows = conn.execute("SELECT feed_url FROM browser_ua_feeds").fetchall()
    return {str(r["feed_url"]) for r in rows}


def _flag_escalation_tier(conn: sqlite3.Connection, table: str, feed_url: str, reason: str) -> bool:
    """INSERT-or-assert one escalation flag. Returns True if newly flagged.

    A flag that is already there is re-asserted rather than ignored: `flagged_at` moves to now (a refusal seen while the re-probe is in
    flight must not be cleared by it), and a manual flag replaces an automatic one's reason so the re-probe leaves it alone."""
    cur = conn.execute(f"INSERT OR IGNORE INTO {table} (feed_url, reason) VALUES (?, ?)", (feed_url, reason[:200]))
    if cur.rowcount > 0:
        return True
    if reason == "manual":
        conn.execute(f"UPDATE {table} SET flagged_at = datetime('now'), reason = 'manual' WHERE feed_url = ?", (feed_url,))
    else:
        conn.execute(f"UPDATE {table} SET flagged_at = datetime('now') WHERE feed_url = ?", (feed_url,))
    return False


def flag_browser_ua_feed(conn: sqlite3.Connection, feed_url: str, reason: str = "") -> bool:
    """Mark a feed for browser-identity fetches. Returns True if newly flagged."""
    feed_url = feed_url.strip()
    if not feed_url:
        return False
    return _flag_escalation_tier(conn, "browser_ua_feeds", feed_url, reason)


def unflag_browser_ua_feed(conn: sqlite3.Connection, feed_url: str) -> None:
    conn.execute("DELETE FROM browser_ua_feeds WHERE feed_url = ?", (feed_url.strip(),))


def get_proxy_feed_urls(conn: sqlite3.Connection) -> set[str]:
    """Feeds whose fetch should route through the outbound proxy in as_needed mode.

    A feed lands here only after browser-UA was already in play (flagged this
    cycle or earlier) and the fetch still failed — see
    `_flag_proxy_feed_on_still_blocked`. The request hook in services.reader_api
    consults this set so the next fetch escalates. Irrelevant in off/always
    mode — those never consult it."""
    rows = conn.execute("SELECT feed_url FROM proxy_feeds").fetchall()
    return {str(r["feed_url"]) for r in rows}


def flag_proxy_feed(conn: sqlite3.Connection, feed_url: str, reason: str = "") -> bool:
    """Mark a feed for as-needed proxy escalation. Returns True if newly flagged."""
    feed_url = feed_url.strip()
    if not feed_url:
        return False
    return _flag_escalation_tier(conn, "proxy_feeds", feed_url, reason)


def unflag_proxy_feed(conn: sqlite3.Connection, feed_url: str) -> None:
    conn.execute("DELETE FROM proxy_feeds WHERE feed_url = ?", (feed_url.strip(),))


def get_tailscale_feed_urls(conn: sqlite3.Connection) -> set[str]:
    """Feeds whose fetch should route through the last-resort proxy in as_needed
    mode.

    A feed lands here only after the primary proxy was already in play (flagged
    this cycle or earlier) and the fetch still failed — see
    `_flag_tailscale_feed_on_still_blocked`. Irrelevant in off/always mode, same
    as proxy_feeds."""
    rows = conn.execute("SELECT feed_url FROM tailscale_feeds").fetchall()
    return {str(r["feed_url"]) for r in rows}


def flag_tailscale_feed(conn: sqlite3.Connection, feed_url: str, reason: str = "") -> bool:
    """Mark a feed for last-resort proxy escalation. Returns True if newly flagged."""
    feed_url = feed_url.strip()
    if not feed_url:
        return False
    return _flag_escalation_tier(conn, "tailscale_feeds", feed_url, reason)


def unflag_tailscale_feed(conn: sqlite3.Connection, feed_url: str) -> None:
    conn.execute("DELETE FROM tailscale_feeds WHERE feed_url = ?", (feed_url.strip(),))


def get_flaresolverr_feed_urls(conn: sqlite3.Connection) -> set[str]:
    """Feeds whose fetch should route through FlareSolverr in as_needed mode.

    A feed lands here only after the primary proxy was already in play and the
    fetch is STILL specifically a bot-challenge — see
    `_flag_flaresolverr_feed_on_still_blocked`. Irrelevant in off/always mode."""
    rows = conn.execute("SELECT feed_url FROM flaresolverr_feeds").fetchall()
    return {str(r["feed_url"]) for r in rows}


def flag_flaresolverr_feed(conn: sqlite3.Connection, feed_url: str, reason: str = "") -> bool:
    """Mark a feed for FlareSolverr escalation. Returns True if newly flagged."""
    feed_url = feed_url.strip()
    if not feed_url:
        return False
    return _flag_escalation_tier(conn, "flaresolverr_feeds", feed_url, reason)


def unflag_flaresolverr_feed(conn: sqlite3.Connection, feed_url: str) -> None:
    conn.execute("DELETE FROM flaresolverr_feeds WHERE feed_url = ?", (feed_url.strip(),))


def _disabled_label(reason: str, disabled_at: str) -> str:
    """ "manual, Oct 12, 2026 3:00am" or "automatic: 410 Gone, Oct 12, 2026 3:00am" for the Inactive Feeds list.

    disabled_at is SQLite's UTC datetime('now'); it is shown in the server's local time like every other timestamp in the UI."""
    who = f"automatic: {reason}" if reason else "manual"
    try:
        when = format_datetime_for_ui(datetime.strptime(disabled_at, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc))
    except ValueError:
        when = None
    return f"{who}, {when}" if when else who


def disable_feed(feed_url: str, reason: str = "", *, claim: bool = True) -> None:
    """Disable a feed (no fetching) without unsubscribing. Pause and Disable are the
    same state: this sets both Lectio's disabled_feeds row AND reader's
    updates_enabled flag, so the Settings tree and Feed Properties agree and the
    scheduler skips it regardless of which one is consulted.

    *reason* is empty when the user paused the feed and a short label when the system did ("410 Gone"). A system reason never
    overwrites an existing row, and a user's own single-feed pause claims a system-disabled feed (so it stops reading as automatic).
    Bulk and triage actions pass claim=False: they add rows but leave an existing automatic reason standing."""
    feed_url = feed_url.strip()
    if not feed_url:
        return
    reason = reason.strip()[:200]
    with get_meta_connection() as conn:
        conn.execute(
            "INSERT OR IGNORE INTO disabled_feeds (feed_url, reason) VALUES (?, ?)",
            (feed_url, reason),
        )
        if not reason and claim:
            conn.execute("UPDATE disabled_feeds SET reason = '' WHERE feed_url = ?", (feed_url,))
    try:
        with get_reader() as reader:
            reader.disable_feed_updates(feed_url)
    except Exception:
        LOGGER.debug("disable_feed: reader.disable_feed_updates failed for %s", feed_url, exc_info=True)
    invalidate_meta_structure_cache()


def enable_feed(feed_url: str) -> None:
    """Re-enable a disabled/paused feed: clears both flags and resets the backoff so
    it's checked on the next cycle (the inverse of disable_feed)."""
    feed_url = feed_url.strip()
    if not feed_url:
        return
    with get_meta_connection() as conn:
        conn.execute("DELETE FROM disabled_feeds WHERE feed_url = ?", (feed_url,))
        # A 410 auto-disable leaves a terminal failure record; re-enabling is the user's call to try again, so drop it rather
        # than show the feed as permanently gone.
        conn.execute("DELETE FROM feed_failure_state WHERE feed_url = ? AND last_error LIKE '410 Gone%'", (feed_url,))
        # Clear the backoff so the feed is checked next cycle, not after the old
        # retry window.
        conn.execute(
            "UPDATE feed_failure_state SET next_retry_at = NULL WHERE feed_url = ?",
            (feed_url,),
        )
    try:
        with get_reader() as reader:
            reader.enable_feed_updates(feed_url)
    except Exception:
        LOGGER.debug("enable_feed: reader.enable_feed_updates failed for %s", feed_url, exc_info=True)
    invalidate_problematic_feeds_cache()
    invalidate_meta_structure_cache()


def get_meta_structure_snapshot(conn: sqlite3.Connection) -> dict[str, object]:
    """Cached bundle of folder/feed structure queries.

    These tables change only on explicit user mutations, so once computed the
    snapshot stays valid until something invalidates it. Collapses ~5 SQL calls
    per home render to a dict lookup, which is what we actually need under
    concurrent navigation: tiny SELECTs that are fine standalone but
    catastrophic when 8 worker threads each issue 5 of them per page render.

    Also builds folder_feed_urls_by_id (root → all feeds; non-root → that
    folder's feeds), exploiting Lectio's flat one-level-below-root hierarchy
    so we don't need a recursive CTE per request.
    """
    with _meta_structure_lock:
        if _meta_structure_cache:
            return dict(_meta_structure_cache)
    raw_folder_rows = get_folder_rows(conn)
    direct_feed_urls_by_folder = get_direct_feed_urls_by_folder(conn)
    folder_options = get_folder_options(conn)
    all_feed_urls = get_all_feed_urls(conn)
    root_id = get_root_folder_id(conn)

    folder_feed_urls_by_id: dict[int, set[str]] = {}
    for row in raw_folder_rows:
        fid = int(row["id"])
        if fid == root_id:
            folder_feed_urls_by_id[fid] = set(all_feed_urls)
        else:
            folder_feed_urls_by_id[fid] = set(direct_feed_urls_by_folder.get(fid, []))

    snapshot: dict[str, object] = {
        "raw_folder_rows": [dict(r) for r in raw_folder_rows],
        "direct_feed_urls_by_folder": {k: list(v) for k, v in direct_feed_urls_by_folder.items()},
        "folder_options": folder_options,
        "all_feed_urls": all_feed_urls,
        "root_id": root_id,
        "folder_feed_urls_by_id": folder_feed_urls_by_id,
    }
    with _meta_structure_lock:
        _meta_structure_cache.update(snapshot)
    return snapshot


def get_unread_counts_by_folder(
    folder_rows: Sequence[sqlite3.Row | dict],
    unread_counts_by_feed: dict[str, int],
    direct_feed_urls_by_folder: dict[int, list[str]],
) -> dict[int, int]:
    counts = {int(cast(int, row["id"])): 0 for row in folder_rows}
    if not counts:
        return counts

    root_folder_id = None
    for row in folder_rows:
        folder_id = int(cast(int, row["id"]))
        depth = int(cast(int, row["depth"]))
        direct_count = sum(unread_counts_by_feed.get(feed_url, 0) for feed_url in direct_feed_urls_by_folder.get(folder_id, []))
        counts[folder_id] = direct_count
        if depth == 0:
            root_folder_id = folder_id

    if root_folder_id is not None:
        counts[root_folder_id] = sum(counts.values())

    return counts


def _compute_unread_counts_by_feed() -> dict[str, int]:
    """Per-feed unread count via a single SQL GROUP BY — one query for all feeds.

    Cross-feed dedupe (same article appearing in RSS+Atom mirrors of one source)
    is intentionally dropped — it required a full Python scan of every entry on
    every cache miss, which doesn't scale past a few thousand entries. Users
    who run into double-counted feeds should unsubscribe one of the mirrors."""
    try:
        conn = sqlite3.connect(str(tenancy.reader_db_path()), uri=False, check_same_thread=False)
        conn.execute("PRAGMA journal_mode=WAL")
        rows = conn.execute("SELECT feed, COUNT(*) FROM entries WHERE read=0 GROUP BY feed").fetchall()
        counts = {str(row[0]): int(row[1]) for row in rows}
        _subtract_hidden_locked_comics_from_counts(conn, counts)
        _subtract_hidden_unpremiered_from_counts(conn, counts)
        conn.close()
        return counts
    except Exception:
        LOGGER.exception("_compute_unread_counts_by_feed direct SQL failed, falling back")
        counts: dict[str, int] = {}
        with get_reader() as reader:
            for feed in reader.get_feeds():
                counts[feed.url] = reader.get_entry_counts(feed=feed.url).unread or 0
        return counts


def _subtract_hidden_locked_comics_from_counts(reader_conn: sqlite3.Connection, counts: dict[str, int]) -> None:
    """A locked comic hide_locked_comics keeps out of the list must not
    inflate the unread badge either — reported live 2026-09-06 as "shows an
    unread for the hidden post, nothing visible in the feed." Mirrors the
    render-time filter's own condition (locked_until in the future), just
    computed as a count-adjustment instead of a per-entry skip, since this
    function only has feed-level totals to work with.

    Two databases, two queries: entry_lead_images.locked_until lives in the
    meta DB, read state lives in reader_conn (already open on the reader DB) —
    nothing here can be a single SQL JOIN. The locked set is normally tiny
    (a handful of feeds at most), so this costs one small meta-DB query plus
    one reader-DB query per feed that actually has a currently-locked entry,
    not a scan of everything.
    """
    try:
        global_hide = hide_locked_comics_global()
        with get_meta_connection() as mconn:
            if global_hide:
                rows = mconn.execute(
                    "SELECT feed_url, entry_id FROM entry_lead_images WHERE locked_until > ?",
                    (time.time(),),
                ).fetchall()
            else:
                per_feed_feeds = {
                    str(r["feed_url"])
                    for r in mconn.execute("SELECT feed_url FROM feed_display_prefs WHERE hide_locked_comics = 1").fetchall()
                }
                if not per_feed_feeds:
                    return
                _feed_list = list(per_feed_feeds)
                rows = []
                for _i in range(0, len(_feed_list), 999):
                    _chunk = _feed_list[_i : _i + 999]
                    _ph = ",".join("?" for _ in _chunk)
                    rows.extend(
                        mconn.execute(
                            f"SELECT feed_url, entry_id FROM entry_lead_images WHERE locked_until > ? AND feed_url IN ({_ph})",
                            [time.time(), *_chunk],
                        ).fetchall()
                    )
        if not rows:
            return
        by_feed: dict[str, list[str]] = {}
        for feed_url, entry_id in rows:
            by_feed.setdefault(str(feed_url), []).append(str(entry_id))
        for feed_url, entry_ids in by_feed.items():
            # Chunked: an unchunked IN binds one parameter per currently-locked
            # entry on this feed (plus the feed_url) and, past SQLite's variable
            # limit, raises -- which the except below would then swallow for the
            # WHOLE function, silently reverting every feed's badge to its raw
            # unread total, not just this one feed's.
            hidden_unread = 0
            for _i in range(0, len(entry_ids), 900):
                _chunk = entry_ids[_i : _i + 900]
                _ph = ",".join("?" for _ in _chunk)
                hidden_unread += reader_conn.execute(
                    f"SELECT COUNT(*) FROM entries WHERE feed = ? AND read = 0 AND id IN ({_ph})",
                    [feed_url, *_chunk],
                ).fetchone()[0]
            if hidden_unread:
                counts[feed_url] = max(0, counts.get(feed_url, 0) - hidden_unread)
    except Exception:
        LOGGER.exception("[hide-locked-comics] failed to subtract hidden counts from unread totals")


def _subtract_hidden_unpremiered_from_counts(reader_conn: sqlite3.Connection, counts: dict[str, int]) -> None:
    """A not-yet-premiered YouTube video hide_unpremiered keeps out of the list
    must not inflate the unread badge either — same gap
    _subtract_hidden_locked_comics_from_counts was built for, identified but not
    fixed alongside it (2026-09-06).

    Unlike locked comics, there is no indexed meta-DB column keyed by
    (feed_url, entry_id) for premiere status: `_is_youtube_unpremiered` works by
    extracting a video id out of the entry's *link* and checking
    youtube_duration_service's cached live status (itself warmed into memory at
    startup, so this is an in-memory lookup per candidate entry, not a DB hit
    per entry). So instead of one meta-DB join, this walks each qualifying
    feed's unread entry links and re-derives the same check the render-time
    filter uses. The candidate feed set is normally tiny (a handful of YouTube
    feeds with the toggle on, at most), so this costs one small query per such
    feed, not a scan of everything.
    """
    try:
        global_hide = youtube_hide_unpremiered_global()
        yt_feeds = [f for f, n in counts.items() if n and "youtube.com/feeds/videos.xml" in f]
        if not yt_feeds:
            return
        if global_hide:
            candidate_feeds = yt_feeds
        else:
            per_feed_on: set[str] = set()
            with get_meta_connection() as mconn:
                for _i in range(0, len(yt_feeds), 900):
                    _chunk = yt_feeds[_i : _i + 900]
                    _ph = ",".join("?" for _ in _chunk)
                    per_feed_on.update(
                        str(r["feed_url"])
                        for r in mconn.execute(
                            f"SELECT feed_url FROM feed_display_prefs WHERE hide_unpremiered = 1 AND feed_url IN ({_ph})",
                            _chunk,
                        ).fetchall()
                    )
            candidate_feeds = [f for f in yt_feeds if f in per_feed_on]
        for feed_url in candidate_feeds:
            hidden_unread = 0
            for (link,) in reader_conn.execute("SELECT link FROM entries WHERE feed = ? AND read = 0", (feed_url,)).fetchall():
                if _youtube_unpremiered_video_id(feed_url, link):
                    hidden_unread += 1
            if hidden_unread:
                counts[feed_url] = max(0, counts.get(feed_url, 0) - hidden_unread)
    except Exception:
        LOGGER.exception("[hide-unpremiered] failed to subtract hidden counts from unread totals")


def _parse_reader_timestamp(ts: str) -> datetime | None:
    """Parse a reader entries timestamp (ISO, possibly space-separated / naive)
    into an aware UTC datetime. Returns None if unparseable."""
    ts = (ts or "").strip()
    if not ts:
        return None
    try:
        dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
    except ValueError:
        try:
            dt = datetime.strptime(ts[:19], "%Y-%m-%d %H:%M:%S")
        except ValueError:
            return None
    if dt.tzinfo is None:
        dt = dt.replace(tzinfo=timezone.utc)
    return dt


def get_feed_last_post_dates() -> dict[str, datetime]:
    """Newest entry timestamp per feed (published, then updated, then first
    seen), read straight from the reader DB in one GROUP BY. Cached per-user
    with a short TTL. Feeds with no entries are simply absent from the map."""
    db_key = str(tenancy.reader_db_path())
    now = time.time()
    with _feed_last_post_cache_lock:
        cached = _feed_last_post_cache.get(db_key)
        if cached and now - cached[0] < FEED_LAST_POST_TTL_SECONDS:
            return dict(cached[1])

    result: dict[str, datetime] = {}
    try:
        conn = sqlite3.connect(db_key, uri=False, check_same_thread=False)
        try:
            rows = conn.execute("SELECT feed, MAX(COALESCE(published, updated, first_updated)) FROM entries GROUP BY feed").fetchall()
        finally:
            conn.close()
    except Exception:
        LOGGER.exception("get_feed_last_post_dates direct SQL failed")
        return result

    for feed_url, ts in rows:
        dt = _parse_reader_timestamp(str(ts)) if ts else None
        if dt is not None:
            result[str(feed_url)] = dt

    with _feed_last_post_cache_lock:
        _feed_last_post_cache[db_key] = (time.time(), dict(result))
    return result


def _refresh_unread_counts_async(generation: int) -> None:
    """Single-flight background scan. Updates cache when done, unless the
    cache was invalidated (generation bumped) after this refresh started."""
    try:
        counts = _compute_unread_counts_by_feed()
        with unread_counts_cache_lock:
            if get_unread_counts_generation() == generation:
                unread_counts_cache["unread_counts"] = (time.time(), counts)
    except Exception:
        LOGGER.exception("background unread counts refresh failed")
    finally:
        clear_unread_refresh_inflight()


def get_unread_counts_by_feed() -> dict[str, int]:
    """Stale-while-revalidate: never block the request on the 33k-entry scan.
    Fresh cache → return it. Stale cache → return stale, kick off background
    refresh. Cold cache → first caller computes synchronously, others wait."""
    now = time.time()
    with unread_counts_cache_lock:
        cached = unread_counts_cache.get("unread_counts")
        current_gen = get_unread_counts_generation()
    if cached:
        ts, value = cached
        if now - ts < UNREAD_COUNTS_CACHE_TTL_SECONDS:
            return value.copy()
        # Stale — serve it, spawn one refresh.
        if try_start_unread_refresh():
            threading.Thread(
                target=_run_in_user_context,
                args=(tenancy.current_user_id(), _refresh_unread_counts_async, current_gen),
                daemon=True,
            ).start()
        return value.copy()

    # Cold cache: first arriver computes synchronously, others wait on lock.
    with unread_counts_compute_lock:
        with unread_counts_cache_lock:
            cached = unread_counts_cache.get("unread_counts")
            if cached:
                return cached[1].copy()
            gen_at_start = get_unread_counts_generation()
        counts = _compute_unread_counts_by_feed()
        with unread_counts_cache_lock:
            # Only cache if no mark-read/refresh bumped the generation while we
            # were computing (the scan takes ~2s). Otherwise `counts` predates
            # that change and would poison the cache with stale unread counts —
            # e.g. mark-older-than-read appearing to revert seconds later.
            if get_unread_counts_generation() == gen_at_start:
                unread_counts_cache["unread_counts"] = (time.time(), counts)
        return counts.copy()


# A stored date at or before this is a MISSING date that arrived as a sentinel,
# not a publication date. Two clusters exist in a real library — the Unix epoch
# (an importer's "0") and year 0001 (a parser's zero-value) — 312 entries across
# 31 feeds when measured 2026-08-04, with nothing at all between year 1 and 1995.
# The cutoff matches the `_MIN_YEAR = 1990` convention the recovery scripts in
# scripts/ already use.
#
# This matters far more than it looks: every date fallback in this file is an
# `or` chain, and a sentinel is *truthy*. A NULL falls through to the next
# source; epoch-0 stops the chain dead and wins. That is why an entry whose URL
# carries its own date still displayed nothing.
_SENTINEL_DATE_BEFORE = datetime(1990, 1, 1, tzinfo=timezone.utc)


def real_published_date(value: datetime | None) -> datetime | None:
    """*value* if it is a plausible publication date, else None.

    Normalizes the missing-date sentinels to the None they should have been, so
    callers can keep writing plain `or` chains.
    """
    if value is None:
        return None
    try:
        # reader stores naive UTC; an aware value may arrive from an override.
        ref = value if value.tzinfo is not None else value.replace(tzinfo=timezone.utc)
        return None if ref < _SENTINEL_DATE_BEFORE else value
    except AttributeError, TypeError, ValueError, OverflowError:
        return None


# SQL mirror of entry_publication_date's FEED-supplied sources, for the list
# fast paths that order thousands of entries without hydrating them.
#
# `updated` is the load-bearing term and was missing. A feed may ship <updated>
# and no <published> — 2,696 entries across 84 feeds here — and
# entry_publication_date reads both, so the SQL ordering disagreed with the
# ordering everything else uses. The prefetch takes the oldest (or newest) N
# rows by ITS key, so a disagreement does not merely misplace an entry, it
# DROPS it: one art feed's entry is dated 2026-07-21 by
# <updated> and 2026-08-12 by first_updated, so it fell outside a window
# spanning 2026-07-20 to 2026-08-01 and vanished from All Feeds while showing
# correctly in its folder (which has few enough feeds to use reader's own
# per-feed query, and reader reads <updated>).
#
# The URL/title inference tiers of entry_publication_date are deliberately not
# reproduced here: they cannot be expressed in SQL, and they only ever apply to
# entries this expression already treats as undated.
_ENTRY_SORT_SQL = "coalesce(published, updated, first_updated)"


def entry_publication_date(entry) -> datetime | None:
    """When the entry was actually published, or None when nothing says.

    Ordered by how much the source can be trusted: the feed's own dates first,
    then a date the permalink states outright, then a month-precision permalink,
    then one parsed out of the title. Deliberately does NOT fall back to the
    received date — "when Lectio first saw it" is not a publication date, and a
    caller that wants that fallback should say so via `entry_effective_date`.
    Keeping them apart is what lets the UI show "no date" honestly.

    `entry.link` and `entry.id` can differ and the date may live in either.
    """
    return (
        real_published_date(getattr(entry, "published", None))
        or real_published_date(getattr(entry, "updated", None))
        or url_inferred_pubdate(getattr(entry, "link", None))
        or url_inferred_pubdate(getattr(entry, "id", None))
        or url_inferred_pubmonth(getattr(entry, "link", None))
        or url_inferred_pubmonth(getattr(entry, "id", None))
        or title_inferred_pubdate(getattr(entry, "title", None))
    )


def entry_effective_date(entry) -> datetime | None:
    """The date the UI treats as an entry's timestamp: publication date (see
    `entry_publication_date`), falling back to the received (`added`) date for
    entries nothing can date.

    Single source of truth so the list render, unread counts, and the bulk
    age actions (mark-older/newer-than) all agree — a mismatch here made
    mark-older skip entries the UI had optimistically marked, so they flashed
    read then reverted.
    """
    return entry_publication_date(entry) or getattr(entry, "added", None)


_DEDUPE_SCHEME_RE = re.compile(r"^[a-z][a-z0-9+.-]*://", re.I)


def normalize_entry_link_for_dedupe(link: str | None, host_aliases: dict[str, str] | None = None) -> str | None:
    """Canonical comparison key for an article link.

    The scheme and a leading ``www.`` are folded away: http/https and
    www/non-www are the same article, and keeping them apart pushed real
    duplicates into the saved scan's weaker "possible" tier whenever the URL
    slug was too generic for `_safe_dedup_entry_slug` to rescue the pair
    (``/index.html``, hyphen-free stubs). Only the host is lowercased — paths
    are case-sensitive.

    ``host_aliases`` (a ``{from_host: to_host}`` map) additionally folds
    user-declared domain migrations so an article saved under an author's old
    host pairs with the same article on the new one; without it the key stays
    strictly host-scoped. See `get_dedupe_host_aliases`.

    The result is a comparison key, not a URL: never fetch or display it.
    """
    if not link:
        return None
    normalized_link = str(link).split("#")[0].strip().rstrip("/")
    if not normalized_link:
        return None
    # Protocol-relative links fold the same way a scheme'd one does.
    stripped = _DEDUPE_SCHEME_RE.sub("", normalized_link).removeprefix("//")
    host, sep, rest = stripped.partition("/")
    host = host.lower()
    if host.startswith("www."):
        host = host[4:]
    if host_aliases:
        host = host_aliases.get(host, host)
    return f"{host}{sep}{rest}" or None


def build_entry_dedupe_key(link: str | None, title: str | None) -> str | None:
    normalized_link = normalize_entry_link_for_dedupe(link)
    if not normalized_link:
        return None
    normalized_title = normalize_entry_title_for_dedupe(title)
    return f"{normalized_link}::{normalized_title}"


def _resolve_dedup_feed_urls(
    conn: sqlite3.Connection,
    scope: str,
    scope_id: str,
    exclude_scope_ids: str = "",
    custom_feed_urls: set[str] | None = None,
) -> set[str] | dict:
    """Resolve the in-scope feed URLs for a dedup run (shared by preview + apply).

    Returns the feed-URL set, or an ``{"error": ...}`` dict for an invalid scope.
    A custom feed set (the manual compare picker) bypasses scope/exclude resolution."""
    if custom_feed_urls is not None:
        return custom_feed_urls
    if scope == "global":
        feed_urls = get_all_feed_urls(conn)
    elif scope == "folder":
        try:
            fid = int(scope_id)
        except ValueError, TypeError:
            return {"error": "invalid scope_id"}
        feed_urls = get_folder_feed_urls(conn, fid)
    elif scope == "folders":
        feed_urls = set()
        for fid in parse_folders_scope_id(scope_id):
            feed_urls |= get_folder_feed_urls(conn, fid)
    elif scope == "feeds":
        # Dedupe across an explicit set of selected feeds (>=2 needed; checked by caller).
        feed_urls = set(parse_feeds_scope_id(scope_id))
    else:
        return {"error": "deduplicate rules require global, folder(s), or multi-feed scope"}
    if exclude_scope_ids:
        excluded: set[str] = set()
        for fid_str in exclude_scope_ids.split(","):
            fid_str = fid_str.strip()
            if fid_str.isdigit():
                excluded |= get_folder_feed_urls(conn, int(fid_str))
        feed_urls -= excluded
    return feed_urls


def split_keyword_terms(keyword: str) -> list[str]:
    """Split a plain (non-regex) keyword into its comma-separated OR terms.

    Multiple terms used to mean writing a regex, and a regex written by hand is
    where the word-boundary traps live (``Apple|iPhone`` also matches "grapples"
    and "pineapple"). A comma list needs no escaping and no boundary syntax.

    Terms keep SUBSTRING semantics, exactly as a single keyword always had:
    measured on the live library, 4 plain rules match only *inside* words and
    would stop matching entirely if the split implied a word boundary.
    A term containing a literal comma needs regex mode.
    """
    return [term for term in (part.strip() for part in keyword.split(",")) if term]


# Publishers spell the same punctuation two ways and the reader cannot see which:
# 2,773 stored titles use a typographic apostrophe, 6,761 the ASCII one. A term
# typed as "Apple's" must match "Apple’s". Folded on BOTH sides of a plain
# compare, so it only ever adds matches. Regex mode is left alone — a pattern
# says what it says.
_MATCH_FOLD = {
    ord("\u2018"): "'",
    ord("\u2019"): "'",
    ord("\u201b"): "'",
    ord("\u2032"): "'",
    ord("\u201c"): '"',
    ord("\u201d"): '"',
    ord("\u201e"): '"',
    ord("\u201f"): '"',
    ord("\u2033"): '"',
    ord("\u00a0"): " ",
    ord("\u2007"): " ",
    ord("\u202f"): " ",
}


def fold_for_match(text: str) -> str:
    """Lowercase and fold curly punctuation for a plain (non-regex) comparison."""
    return text.translate(_MATCH_FOLD).lower()


def build_keyword_matcher(keyword: str, is_regex: bool):
    """Text -> bool matcher for a rule's keyword. Raises re.error on a bad regex.

    Shared by the dry-run, Run Now and live-refresh paths, which each built their
    own copy of this and could drift apart.
    """
    if is_regex:
        pattern = re.compile(keyword, re.IGNORECASE)
        return lambda text: bool(pattern.search(text)) if text else False
    terms = [fold_for_match(term) for term in split_keyword_terms(keyword)]
    if len(terms) == 1:
        only = terms[0]
        return lambda text: only in fold_for_match(text or "")

    def _match_any(text: str) -> bool:
        # Fold the body ONCE, not once per term: translate() costs more than the
        # old .lower(), and a rule with a dozen terms otherwise re-walks the whole
        # entry body a dozen times.
        folded = fold_for_match(text or "")
        return any(term in folded for term in terms)

    return _match_any


def parse_tag_filter_spec(raw: str | None) -> tuple[set[str], set[str], set[str]]:
    """Parse a tag-filter spec into (require, good, exclude) normalized sets.

    Comma-separated tokens with three strengths:
      ``-tag``  exclude — blocks the entry;
      ``+tag``  (or bare) good — rescues an entry from excludes, but its
                absence never cuts anything;
      ``++tag`` require — tagged entries WITHOUT any required tag are cut
                (whitelist mode, opt-in).
    Commas (not spaces) separate, so multi-word tags can be typed as-is —
    ``+windows 11, -rust`` — and normalize_tag_value hyphenates them to
    match the stored form.
    """
    require: set[str] = set()
    good: set[str] = set()
    exclude: set[str] = set()
    for token in (raw or "").split(","):
        token = token.strip()
        if not token:
            continue
        if token.startswith("++"):
            target, token = require, token[2:]
        elif token[0] == "-":
            target, token = exclude, token[1:]
        elif token[0] == "+":
            target, token = good, token[1:]
        else:
            target = good
        normalized = normalize_tag_value(token)
        if normalized:
            target.add(normalized)
    return require, good, exclude


def author_filter_token(author: str | None) -> str | None:
    """The pseudo-tag a tag_filter rule matches an entry's author under —
    'Steven Parker' → 'by-steven-parker'. Lets '-by-steven-parker' drop an
    author's posts through the same rule/chips machinery as real feed tags."""
    if not author or not str(author).strip():
        return None
    return normalize_tag_value("by " + str(author))


def get_feed_tag_filter_rule(conn: sqlite3.Connection, feed_url: str) -> dict | None:
    """The feed-scoped tag_filter rule for feed_url (the one the post-header
    chips manage), or None. Folder/global tag_filter rules are left alone."""
    row = conn.execute(
        "SELECT rowid, keyword, enabled FROM highlight_keywords WHERE type = 'tag_filter' AND scope = 'feed' AND scope_id = ?",
        (feed_url,),
    ).fetchone()
    return dict(row) if row else None


def get_inherited_tag_filter_signs(conn: sqlite3.Connection, feed_url: str) -> dict[str, str]:
    """tag -> '+'/'-' from ENABLED folder/global/multi-feed tag_filter rules covering feed_url. Display-only: the post-header chips
    color these apart from the feed's own rule, so a filter set at a broader level is visible. Clicking still edits the feed rule.

    The feed's own rule wins the display even while disabled: a chip click creates that rule disabled (armed later in Automation),
    so the chips have to show its state for the click to visibly land."""
    rows = conn.execute(
        "SELECT keyword, scope, scope_id FROM highlight_keywords WHERE type = 'tag_filter' AND scope != 'feed' AND enabled = 1"
    ).fetchall()
    out: dict[str, str] = {}
    for row in rows:
        scope, scope_id = str(row["scope"]), str(row["scope_id"] or "")
        folder_feeds: set[str] | None = None
        if scope in ("folder", "folders"):
            folder_feeds = set()
            for fid in rule_scope_folder_ids(scope, scope_id):
                folder_feeds |= get_folder_feed_urls(conn, fid)
        if not feed_in_rule_scope(scope, scope_id, feed_url, folder_feeds):
            continue
        req, good, exc = parse_tag_filter_spec(str(row["keyword"] or ""))
        for tag in req | good:
            out.setdefault(tag, "+")
        for tag in exc:
            out.setdefault(tag, "-")
    return out


def toggle_feed_tag_filter(conn: sqlite3.Connection, feed_url: str, tag: str, sign: str) -> dict:
    """Toggle a signed tag on the feed's tag_filter rule from the post-header
    chips. Same sign already present → remove it; opposite sign → flip it;
    otherwise append. Creates the rule DISABLED on first use — the chips are a
    tuning surface; the user arms the rule in Automation when it's ready — and
    deletes it when the spec empties. The new spec is applied to unread
    entries immediately only when the rule is already enabled.
    Returns {spec, active, enabled, applied_count}."""
    normalized = normalize_tag_value(tag)
    if not normalized or sign not in ("+", "-"):
        return {"error": "invalid tag"}

    rule = get_feed_tag_filter_rule(conn, feed_url)
    # (sign, normalized) in spec order; sign is '-', '+', or '++' — hand-written
    # require tokens survive chip edits, and the ▲ chip treats them as its own
    # (clicking ▲ on a ++tag removes it rather than downgrading it).
    tokens: list[tuple[str, str]] = []
    if rule:
        for token in str(rule["keyword"] or "").split(","):
            token = token.strip()
            if not token:
                continue
            if token.startswith("++"):
                token_sign = "++"
            elif token[0] == "-":
                token_sign = "-"
            else:
                token_sign = "+"
            value = normalize_tag_value(token.lstrip("+-"))
            if value and all(v != value for _s, v in tokens):
                tokens.append((token_sign, value))

    existing_sign = next((s for s, v in tokens if v == normalized), None)
    same = existing_sign == sign or (sign == "+" and existing_sign == "++")
    if existing_sign is not None and same:
        tokens.remove((existing_sign, normalized))  # toggle off
    else:
        if existing_sign is not None:
            tokens.remove((existing_sign, normalized))  # flip
        tokens.append((sign, normalized))

    spec = ", ".join(f"{s}{v}" for s, v in tokens)
    enabled = bool(rule and rule["enabled"])
    if rule and not tokens:
        conn.execute("DELETE FROM highlight_keywords WHERE rowid = ?", (rule["rowid"],))
    elif rule:
        conn.execute(
            "UPDATE highlight_keywords SET keyword = ? WHERE rowid = ?",
            (spec, rule["rowid"]),
        )
    elif tokens:
        add_highlight_keyword(conn, "feed", feed_url, spec, "yellow", rule_type="tag_filter", enabled=0)

    applied = 0
    if tokens and enabled:
        result = _run_tag_filter(conn, "feed", feed_url, spec)
        if "error" not in result:
            applied = int(result.get("count", 0))
            if applied > 0:
                _log_auto_run(conn, datetime.now().isoformat(), "tag_filter", "feed", feed_url, spec, result, trigger="manual")

    return {"spec": spec, "active": {v: ("+" if s == "++" else s) for s, v in tokens}, "enabled": enabled, "applied_count": applied}


_SHORTS_HASHTAGS = ("#shorts", "#short", "#ytshorts", "#youtubeshorts")


def _is_youtube_short(entry: object) -> bool:
    """Return True if the entry is a YouTube Short.

    YouTube channel feeds use watch?v= links for all videos, so /shorts/ in the
    link is rare. Detect via three signals:
    1. /shorts/ in the link (direct shorts URLs, playlist feeds).
    2. #shorts/#short hashtag in title or description (creator-applied tag;
       YouTube surfaces Shorts on the Shorts shelf when creators add this).
    3. Cached duration ≤ 60 s (in-memory lookup, no I/O).
    """
    link = str(getattr(entry, "link", None) or "")
    if "youtube.com/shorts/" in link:
        return True

    title = (getattr(entry, "title", None) or "").lower()
    summary = (getattr(entry, "summary", None) or "").lower()
    content_text = "".join((getattr(c, "value", None) or "").lower() for c in (getattr(entry, "content", None) or []))
    for text in (title, summary, content_text):
        if any(tag in text for tag in _SHORTS_HASHTAGS):
            return True

    vid = youtube_duration_service.extract_video_id(link)
    if vid:
        dur, _ = youtube_duration_service.get_cached_duration(vid)
        if dur is not None and dur <= 60:
            return True

    return False


def _youtube_unpremiered_video_id(feed_url: str | None, link: str | None) -> str | None:
    """Return the video id if (feed_url, link) is a YouTube video that hasn't
    premiered/gone live yet (liveBroadcastContent == "upcoming"), else None.
    A stream already live, or a normal published video, is not "unpremiered".

    Works from raw (feed_url, link) strings — not just a reader Entry — so the
    same check covers both the entry-object bulk-mark-read loops and the raw
    SQL rows _prune_entries reads."""
    if not link or not feed_url or "youtube.com/feeds/videos.xml" not in str(feed_url):
        return None
    vid = youtube_duration_service.extract_video_id(str(link))
    if not vid:
        return None
    live_status, _ = youtube_duration_service.get_cached_live_status(vid)
    return vid if live_status == "upcoming" else None


def _is_youtube_unpremiered(entry: object) -> bool:
    """True if entry is a YouTube video that hasn't premiered/gone live yet."""
    return _youtube_unpremiered_video_id(getattr(entry, "feed_url", None), getattr(entry, "link", None)) is not None


def _youtube_live_info(entry: object) -> tuple[bool, datetime | None]:
    """``(still_upcoming, scheduled_start)`` from the cached live status of a YouTube entry; ``(False, None)`` for anything else."""
    feed_url, link = getattr(entry, "feed_url", None), getattr(entry, "link", None)
    if not link or not feed_url or "youtube.com/feeds/videos.xml" not in str(feed_url):
        return False, None
    vid = youtube_duration_service.extract_video_id(str(link))
    if not vid:
        return False, None
    live_status, scheduled = youtube_duration_service.get_cached_live_status(vid)
    start = _parse_stored_dt(scheduled)
    if start is not None and start.tzinfo is None:
        start = start.replace(tzinfo=timezone.utc)
    return live_status == "upcoming", start


def _youtube_premiere_prefix(video_id: str) -> str | None:
    """'[Premieres in Xd]'-style title prefix for a video still scheduled to
    air, or None if it isn't an upcoming premiere. Computed live from the
    cached scheduled time on every render, so it self-corrects if YouTube
    moves the date instead of drifting like a baked-in "Live in N days"
    string would."""
    live_status, scheduled_start_time = youtube_duration_service.get_cached_live_status(video_id)
    if live_status != "upcoming":
        return None
    when = _parse_stored_dt(scheduled_start_time)
    if when is None:
        return "Premieres soon"
    if when.tzinfo is None:
        when = when.replace(tzinfo=timezone.utc)
    delta = when - datetime.now(timezone.utc)
    if delta.total_seconds() <= 0:
        return "Premieres soon"
    if delta.days >= 1:
        return f"Premieres in {delta.days}d"
    hours = int(delta.total_seconds() // 3600)
    if hours >= 1:
        return f"Premieres in {hours}h"
    return "Premieres soon"


def _mark_existing_shorts_read(feed_urls: Iterable[str]) -> int:
    """Mark every currently-unread YouTube Short in ``feed_urls`` as read.

    Shared by the after-refresh hide-shorts pass and the per-feed Hide-Shorts
    toggle (so flipping it on immediately clears the backlog of Shorts, not just
    future ones). Returns the number of entries marked read."""
    feed_urls = set(feed_urls)
    if not feed_urls:
        return 0
    now_str = datetime.now().isoformat()
    to_mark: list[tuple[str, str]] = []
    with get_reader() as reader:
        for feed_url in feed_urls:
            for entry in reader.get_entries(feed=feed_url, read=False):
                if _is_youtube_short(entry):
                    to_mark.append((str(entry.feed_url), str(entry.id)))
        for fu, eid in to_mark:
            reader.mark_entry_as_read((fu, eid))
    if to_mark:
        with get_meta_connection() as conn:
            conn.executemany(
                "INSERT INTO entry_read_state (feed_url, entry_id, read_at) VALUES (?,?,?)"
                " ON CONFLICT(feed_url, entry_id) DO UPDATE SET read_at=excluded.read_at",
                [(fu, eid, now_str) for fu, eid in to_mark],
            )
        _bump_unread_counts_generation()
    return len(to_mark)


def _cleanup_intra_feed_slug_dupes(reader, conn) -> int:
    return _cleanup_intra_feed_dupes(
        reader,
        conn,
        is_saved_feed=saved_articles_service.is_saved_articles_feed,
        normalize_link=normalize_entry_link_for_dedupe,
    )


def _apply_hide_shorts(refreshed_feed_urls: set[str]) -> int:
    """Auto-mark Shorts as read for YouTube feeds that have hide-shorts enabled
    (per-feed pref, or every refreshed YouTube feed when the global toggle is on).

    Called from _run_automation_after_refresh right after entries land (link/hashtag
    signals) and again after duration enhancement so ≤60s videos without a #shorts
    tag are also caught. Returns the number of entries marked read."""

    def _is_yt_host(u: str) -> bool:
        host = urlparse(u).hostname or ""
        return host == "youtube.com" or host.endswith(".youtube.com")

    if not any(_is_yt_host(u) for u in refreshed_feed_urls):
        return 0
    try:
        with get_meta_connection() as conn:
            shorts_urls = {
                str(r["feed_url"]) for r in conn.execute("SELECT feed_url FROM feed_display_prefs WHERE hide_shorts = 1").fetchall()
            }
        shorts_targets = refreshed_feed_urls & shorts_urls
        if youtube_hide_shorts_global():
            shorts_targets = shorts_targets | {u for u in refreshed_feed_urls if "youtube.com/feeds/videos.xml" in u}
        if shorts_targets:
            return _mark_existing_shorts_read(shorts_targets)
    except Exception:
        LOGGER.exception("[automation] error applying hide-shorts")
    return 0


# A paywalled post's body is a stub: below this much visible text, and with its
# only link pointing back at the post itself, there is no article here.
_PAYWALL_STUB_MAX_TEXT = 120


def is_paywall_stub(content_html: str | None, entry_link: str | None) -> bool:
    """Is this entry body a subscriber-only stub rather than an article?

    Substack marks paid posts nowhere in its feed — no category, no audience
    field. What it does is ship a body containing only a "Read more" link back to
    the post. Measured on abortretry.fail: 17 of 20 items were 9-character stubs
    ("Read more") against three real posts of 19k-38k characters, so the
    separation is not marginal.

    Requiring the link to point at the ENTRY'S OWN URL is what keeps this from
    firing on a short real post: a two-line link roundup has links that go
    elsewhere.
    """
    if not content_html:
        return False
    text = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", content_html)).strip()
    if len(text) > _PAYWALL_STUB_MAX_TEXT:
        return False
    hrefs = re.findall(r'<a\b[^>]*\bhref\s*=\s*["\']([^"\']+)', content_html, re.IGNORECASE)
    if not hrefs:
        return False
    own = (entry_link or "").split("?", 1)[0].rstrip("/")
    return bool(own) and any(h.split("?", 1)[0].rstrip("/") == own for h in hrefs)


def _min_positive(*values: int) -> int:
    """The smallest value above zero, or 0 when none is (zero means "no limit" at every level)."""
    positive = [v for v in values if v and v > 0]
    return min(positive) if positive else 0


def _int_setting(raw: str | None) -> int:
    try:
        return max(0, int((raw or "").strip() or 0))
    except ValueError:
        return 0


def instance_auto_read_days() -> int:
    return _int_setting(get_instance_setting(SETTING_AUTO_READ_DAYS_INSTANCE))


def user_auto_read_days() -> int:
    return _int_setting(get_runtime_setting(SETTING_AUTO_READ_DAYS_USER))


def folder_auto_read_days(conn: sqlite3.Connection, feed_url: str) -> int:
    """The auto-read age the feed's folders ask for: the smallest positive value across them (a feed in two folders gets the stricter),
    0 when none sets one."""
    row = conn.execute(
        "SELECT MIN(f.auto_read_days) FROM folder_feeds ff JOIN folders f ON f.id = ff.folder_id"
        " WHERE ff.feed_url = ? AND f.auto_read_days > 0",
        (feed_url,),
    ).fetchone()
    return int(row[0] or 0)


def auto_read_parent_cap(conn: sqlite3.Connection | None = None, *, feed_url: str | None = None) -> int:
    """The longest auto-read age a level may choose: the instance and account settings, plus the feed's folders when *feed_url* is given.
    0 = nothing above it sets one."""
    folder = folder_auto_read_days(conn, feed_url) if conn is not None and feed_url else 0
    return _min_positive(instance_auto_read_days(), user_auto_read_days(), folder)


def effective_auto_read_days(conn: sqlite3.Connection, feed_url: str) -> int:
    """Days after which this feed's new posts are auto-marked read; 0 = never. Every level (instance, account, folder, feed) can only
    shorten what the one above allows, so the shortest one set wins -- and a later-lowered parent still holds over a stale child."""
    row = conn.execute("SELECT auto_read_days FROM feed_display_prefs WHERE feed_url = ?", (feed_url,)).fetchone()
    own = int(row[0]) if row is not None and row[0] is not None else 0
    return _min_positive(auto_read_parent_cap(conn, feed_url=feed_url), own)


# Only posts that arrived this recently are swept, so a post the user marks unread later is not re-marked on every refresh.
_AUTO_READ_AGE_ARRIVAL_WINDOW = timedelta(days=1)


def _apply_auto_read_age(refreshed_feed_urls: set[str]) -> int:
    """Auto-mark newly arrived posts read when their date is older than the feed's (or folder's) auto-read age.

    A feed that goes quiet for months and then updates dumps its stale posts into the unread list; this keeps them findable under All
    without making you clear them. Judged by the same effective date the list greys on; a post with no usable date is left unread.
    """
    try:
        with get_meta_connection() as conn:
            ages = {u: effective_auto_read_days(conn, u) for u in refreshed_feed_urls}
        ages = {u: d for u, d in ages.items() if d > 0}
        if not ages:
            return 0
        now = datetime.now(timezone.utc)
        arrived_after = now - _AUTO_READ_AGE_ARRIVAL_WINDOW
        # A supporter-locked webcomic strip is viewable from its unlock time, not its post date.
        with get_meta_connection() as conn:
            _ph = ",".join("?" * len(ages))
            unlock_at = {
                (str(r["feed_url"]), str(r["entry_id"])): datetime.fromtimestamp(float(r["locked_until"]), timezone.utc)
                for r in conn.execute(
                    (
                        "SELECT feed_url, entry_id, locked_until FROM entry_lead_images"
                        f" WHERE locked_until IS NOT NULL AND feed_url IN ({_ph})"
                    ),
                    tuple(ages),
                ).fetchall()
            }
        to_mark: list[tuple[str, str]] = []
        with get_reader() as reader:
            for feed_u, days in ages.items():
                cutoff = now - timedelta(days=days)
                for entry in reader.get_entries(feed=feed_u, read=False):
                    # An entry whose read state was ever changed by hand (marked unread again, say) is the user's call, not ours.
                    if entry.read_modified is not None:
                        continue
                    added = entry.added
                    if added is not None:
                        if added.tzinfo is None:
                            added = added.replace(tzinfo=timezone.utc)
                        if added < arrived_after:
                            continue
                    date = entry_effective_date(entry)
                    if date is None:
                        continue
                    if date.tzinfo is None:
                        date = date.replace(tzinfo=timezone.utc)
                    # Age runs from when the post became viewable, not from the date it showed up: a premiere from its scheduled start, a
                    # locked strip from its unlock. One that isn't viewable yet is left alone entirely.
                    upcoming, premiere_start = _youtube_live_info(entry)
                    unlock = unlock_at.get((feed_u, str(entry.id)))
                    if upcoming or (unlock is not None and unlock > now):
                        continue
                    viewable = max(d for d in (date, premiere_start, unlock) if d is not None)
                    if viewable < cutoff:
                        to_mark.append((feed_u, str(entry.id)))
            for feed_u, entry_id in to_mark:
                reader.mark_entry_as_read((feed_u, entry_id))
        if to_mark:
            when = datetime.now().isoformat()
            with get_meta_connection() as conn:
                conn.executemany(
                    "INSERT INTO entry_read_state (feed_url, entry_id, read_at) VALUES (?, ?, ?)"
                    " ON CONFLICT(feed_url, entry_id) DO UPDATE SET read_at = excluded.read_at",
                    [(fu, eid, when) for fu, eid in to_mark],
                )
            invalidate_unread_counts_cache()
            LOGGER.info("[automation] auto-read age marked %d old post(s) read", len(to_mark))
        return len(to_mark)
    except Exception:
        LOGGER.exception("[automation] error applying auto-read age")
        return 0


def _apply_hide_paywalled(refreshed_feed_urls: set[str]) -> int:
    """Auto-mark subscriber-only stubs read on feeds with hide-paywalled enabled.

    Marks read rather than filtering the list, exactly as hide-shorts does: the
    posts stay findable under All, and nothing is deleted for a paywall that might
    lift. Per feed and opt-in, because a *partial* feed is all stubs by design —
    switching this on there would empty it.
    """
    try:
        with get_meta_connection() as conn:
            targets = {
                str(r["feed_url"]) for r in conn.execute("SELECT feed_url FROM feed_display_prefs WHERE hide_paywalled = 1").fetchall()
            } & set(refreshed_feed_urls)
        if not targets:
            return 0
        to_mark: list[tuple[str, str]] = []
        with get_reader() as reader:
            for feed_u in targets:
                for entry in reader.get_entries(feed=feed_u, read=False):
                    body = entry_content.entry_body(entry)
                    if is_paywall_stub(body, str(entry.link or "")):
                        to_mark.append((feed_u, str(entry.id)))
            for feed_u, entry_id in to_mark:
                reader.mark_entry_as_read((feed_u, entry_id))
        if to_mark:
            # One batched write instead of upsert_entry_read_state per entry --
            # same fix, and same shape, as _mark_existing_shorts_read's sibling
            # pass (see Plan.md Tier 1 refresh-contention item): a feed with many
            # paywalled stubs was taking the meta writer lock once per entry
            # during an active refresh pass instead of once per feed.
            when = datetime.now().isoformat()
            with get_meta_connection() as conn:
                conn.executemany(
                    "INSERT INTO entry_read_state (feed_url, entry_id, read_at) VALUES (?, ?, ?)"
                    " ON CONFLICT(feed_url, entry_id) DO UPDATE SET read_at = excluded.read_at",
                    [(fu, eid, when) for fu, eid in to_mark],
                )
            invalidate_unread_counts_cache()
            LOGGER.info("[automation] hide-paywalled marked %d stub(s) read", len(to_mark))
        return len(to_mark)
    except Exception:
        LOGGER.exception("[automation] error applying hide-paywalled")
        return 0


def _apply_hide_members_only(refreshed_feed_urls: set[str]) -> int:
    """Auto-mark members-only videos read on YouTube feeds that have
    hide-members-only enabled (per-feed pref, or every refreshed YouTube feed
    when the global toggle is on).

    Unlike hide-shorts/hide-unpremiered, membership status isn't in the
    videos.list response duration/premiere detection already fetched — the
    Data API has no field for it — so each never-before-checked video costs a
    real watch-page fetch (fetch_and_cache_members_only). That's still bounded:
    a video is only ever checked once (the result is cached permanently), and
    only for entries on feeds that opted in.
    """

    def _is_yt_host(u: str) -> bool:
        host = urlparse(u).hostname or ""
        return host == "youtube.com" or host.endswith(".youtube.com")

    if not any(_is_yt_host(u) for u in refreshed_feed_urls):
        return 0
    try:
        with get_meta_connection() as conn:
            members_only_urls = {
                str(r["feed_url"]) for r in conn.execute("SELECT feed_url FROM feed_display_prefs WHERE hide_members_only = 1").fetchall()
            }
        # Scoped to YouTube hosts even for the per-feed pref: hide_members_only is a
        # plain feed_display_prefs column with no host constraint of its own, and the
        # Feed Properties checkbox only being shown on YouTube feeds doesn't stop the
        # value existing on a non-YouTube one (a bulk settings-copy, a stray API call).
        # Without this, a mixed refresh batch scans that feed's entries too, and any
        # entry.link that happens to parse as a YouTube URL gets a real watch-page
        # fetch and, if flagged, marked read -- on a feed the user never opted in.
        targets = {u for u in (refreshed_feed_urls & members_only_urls) if _is_yt_host(u)}
        if youtube_hide_members_only_global():
            targets = targets | {u for u in refreshed_feed_urls if "youtube.com/feeds/videos.xml" in u}
        if not targets:
            return 0
        to_mark: list[tuple[str, str]] = []
        with get_reader() as reader:
            for feed_u in targets:
                for entry in reader.get_entries(feed=feed_u, read=False):
                    video_id = youtube_duration_service.extract_video_id(str(entry.link or ""))
                    if not video_id:
                        continue
                    is_members_only = youtube_duration_service.get_cached_members_only(video_id)
                    if is_members_only is None:
                        is_members_only = youtube_duration_service.fetch_and_cache_members_only(video_id)
                    if is_members_only:
                        to_mark.append((feed_u, str(entry.id)))
            for feed_u, entry_id in to_mark:
                reader.mark_entry_as_read((feed_u, entry_id))
        if to_mark:
            # One batched write instead of upsert_entry_read_state per entry --
            # same fix, and same shape, as _apply_hide_paywalled's sibling pass.
            when = datetime.now().isoformat()
            with get_meta_connection() as conn:
                conn.executemany(
                    "INSERT INTO entry_read_state (feed_url, entry_id, read_at) VALUES (?, ?, ?)"
                    " ON CONFLICT(feed_url, entry_id) DO UPDATE SET read_at = excluded.read_at",
                    [(fu, eid, when) for fu, eid in to_mark],
                )
            invalidate_unread_counts_cache()
            LOGGER.info("[automation] hide-members-only marked %d video(s) read", len(to_mark))
        return len(to_mark)
    except Exception:
        LOGGER.exception("[automation] error applying hide-members-only")
        return 0


def _is_local_dev_feed(feed_url: str) -> bool:
    """Return True for feeds served by Lectio itself (bypass refresh cooldown)."""
    try:
        return urlparse(feed_url).path.startswith("/dev/feeds/")
    except Exception:
        return False


def email_lead_image_url(feed_url: str, entry_id: str, link: str) -> str | None:
    """The raw article lead image for an email hero or digest thumbnail. The list thumbnail is not a fallback — it can be a cropped
    panel or a promoted variant — except for YouTube, whose computed "thumbnail" is the full video frame."""
    lead = lead_image_service.get_cached_lead_image_url(feed_url, entry_id)
    if not lead and "youtube.com/feeds/videos.xml" in feed_url:
        lead = lead_image_service.get_cached_entry_thumbnail(feed_url, entry_id, link)
    return lead or None


def email_article_extras(feed_url: str, entry_id: str, entry) -> dict:
    """What makes a shared article look like the entry pane: its lead image, author, and date. Keyword args for send_article_email."""
    published = entry_effective_date(entry)
    return {
        "lead_image_url": email_lead_image_url(feed_url, entry_id, str(getattr(entry, "link", None) or "")),
        "author": str(getattr(entry, "authors_str", None) or "").strip() or None,
        "published": f"{published:%b} {published.day}, {published:%Y}" if published else None,
    }


def _flush_email_batch_for_rule(
    conn: "sqlite3.Connection",
    scope: str,
    scope_id: str,
    keyword: str,
    email_to: str,
    cc_addr: str | None,
    now_str: str,
    *,
    rule_uid: str = "",
) -> None:
    """Send a digest email for one rule's queued entries and clear the queue.

    Selected by rule_uid when given (non-empty) rather than the scope/keyword
    text tuple, so a rule edited mid-backlog still flushes as one digest —
    entries queued before and after the edit carry the same stable rule_uid
    even though their stored rule_scope/rule_keyword text differs. scope/
    scope_id/keyword are still used for the run-log entry either way.
    """
    if rule_uid:
        rows = conn.execute(
            "SELECT id, feed_url, entry_id, title, link, feed_title, excerpt, cc_me FROM email_batch_queue WHERE rule_uid=? AND email_to=?",
            (rule_uid, email_to),
        ).fetchall()
    else:
        rows = conn.execute(
            "SELECT id, feed_url, entry_id, title, link, feed_title, excerpt, cc_me FROM email_batch_queue"
            " WHERE rule_scope=? AND rule_scope_id=? AND rule_keyword=? AND email_to=?",
            (scope, scope_id, keyword, email_to),
        ).fetchall()
    if not rows:
        return
    articles = [
        {
            "title": r["title"],
            "link": r["link"],
            "feed_title": r["feed_title"],
            "excerpt": r["excerpt"],
            "lead_image_url": email_lead_image_url(r["feed_url"], r["entry_id"], str(r["link"] or "")) if r["entry_id"] else None,
        }
        for r in rows
    ]
    use_cc = cc_addr if any(r["cc_me"] for r in rows) else None
    ok, err = send_digest_email(
        get_resend_api_key(),
        get_resend_from(),
        email_to,
        articles,
        cc_addr=use_cc,
    )
    if ok:
        ids = [r["id"] for r in rows]
        placeholders = ",".join("?" * len(ids))
        conn.execute(f"DELETE FROM email_batch_queue WHERE id IN ({placeholders})", ids)
        _log_auto_run(
            conn,
            now_str,
            "email_article",
            scope,
            scope_id,
            keyword,
            {
                "count": len(articles),
                "entries": [
                    {"feed_url": "", "entry_id": "", "title": a["title"], "link": a["link"], "feed_title": a["feed_title"]}
                    for a in articles
                ],
            },
        )
    else:
        LOGGER.warning("[email-auto] digest send failed for %s/%s: %s", scope, keyword, err)


def _flush_all_email_batches() -> None:
    """Flush all pending batch queues — called by daily maintenance as a safety net.

    Grouped by rule_uid when rows carry one, so a rule edited mid-backlog
    flushes as a single digest instead of splitting into one email per text
    identity its scope/keyword happened to have while rows were queued (the
    old GROUP BY rule_scope/rule_scope_id/rule_keyword did exactly that split).
    Rows queued before rule_uid existed keep '' and fall back to the text
    tuple, same as youtube_playlist_added's equivalent rows.
    """
    if not is_email_configured():
        return
    try:
        with get_meta_connection() as conn:
            profile_email = get_setting(conn, PROFILE_EMAIL_SETTING_KEY) or ""
            groups = conn.execute(
                "SELECT rule_uid, MIN(rule_scope) as rule_scope, MIN(rule_scope_id) as rule_scope_id,"
                " MIN(rule_keyword) as rule_keyword, email_to, MAX(cc_me) as cc_me"
                " FROM email_batch_queue"
                " GROUP BY (CASE WHEN rule_uid != '' THEN rule_uid"
                " ELSE 'legacy:' || rule_scope || ':' || rule_scope_id || ':' || rule_keyword END), email_to",
            ).fetchall()
        for g in groups:
            cc_addr = profile_email if g["cc_me"] and profile_email else None
            with get_meta_connection() as conn:
                _flush_email_batch_for_rule(
                    conn,
                    str(g["rule_scope"]),
                    str(g["rule_scope_id"]),
                    str(g["rule_keyword"]),
                    str(g["email_to"]),
                    cc_addr,
                    datetime.now().isoformat(),
                    rule_uid=str(g["rule_uid"] or ""),
                )
    except Exception:
        LOGGER.exception("[email-auto] error flushing all email batches")


def _check_and_flush_batch_times() -> None:
    """Flush email batch queues whose batch_time matches the current local HH:MM.

    Called every minute by the maintenance loop. Each rule's batch_time is a
    "HH:MM" string (24-hour, local time); when the clock matches, the pending
    queue for that rule is flushed and an email digest is sent.
    """
    if not is_email_configured():
        return
    try:
        now_hhmm = time.strftime("%H:%M")
        with get_meta_connection() as conn:
            all_rules = get_highlight_keywords(conn)
            profile_email = get_setting(conn, PROFILE_EMAIL_SETTING_KEY) or ""
        for rule in all_rules:
            if not (rule.get("enabled") and rule.get("delivery") == "batch"):
                continue
            bt = str(rule.get("batch_time") or "").strip()
            if bt != now_hhmm:
                continue
            email_to = str(rule.get("email_to") or "").strip()
            if not email_to:
                continue
            scope = str(rule.get("scope", ""))
            scope_id = str(rule.get("scope_id") or "")
            keyword = str(rule.get("keyword", ""))
            rule_uid = str(rule.get("rule_uid") or "")
            cc_me = bool(rule.get("cc_me"))
            cc_addr = profile_email if cc_me and profile_email and profile_email.lower() != email_to.lower() else None
            with get_meta_connection() as conn:
                _flush_email_batch_for_rule(
                    conn,
                    scope,
                    scope_id,
                    keyword,
                    email_to,
                    cc_addr,
                    datetime.now().isoformat(),
                    rule_uid=rule_uid,
                )
            LOGGER.info("[email-batch] flushed batch for %s/%s at %s", scope, keyword, now_hhmm)
    except Exception:
        LOGGER.exception("[email-batch] error in _check_and_flush_batch_times")


_reader_thread_local = threading.local()

# Cap on distinct users' Reader handles kept open per worker thread. Reader open
# is expensive, so we persist handles; but with many users a thread could
# accumulate one connection per user, so we LRU-evict (and close) the least
# recently used beyond this bound. Generous for the small-tenant target; a
# handful of users never hit it.
_READER_POOL_MAX_PER_THREAD = 8


def close_thread_db_pools() -> None:
    """Close and clear the current thread's pooled reader/meta DB handles.

    Used by test fixtures that swap the tenancy layout between temp dirs:
    just nulling the pools leaks open SQLite handles until GC, and enough
    leaked finalizers across a run make an unrelated later open fail with
    'database is locked' (the flaky-CI signature)."""
    pool = getattr(_reader_thread_local, "pool", None)
    if pool:
        for proxy in list(pool.values()):
            try:
                proxy._reader.close()
            except Exception:  # noqa: BLE001
                pass
    _reader_thread_local.pool = None
    mpool = getattr(_meta_conn_local, "pool", None)
    if mpool:
        for conn in list(mpool.values()):
            try:
                conn.close()
            except Exception:  # noqa: BLE001
                pass
    _meta_conn_local.pool = None


class _PersistentReaderProxy:
    """Wraps a thread-persistent Reader so the existing
    ``with get_reader() as r:`` pattern works without actually closing the
    underlying connections on exit. Reader open is expensive (schema /
    migration checks); paying it once per worker thread instead of per
    request avoids a major source of contention under load."""

    __slots__ = ("_reader",)

    def __init__(self, reader):
        self._reader = reader

    def __enter__(self):
        return self._reader

    def __exit__(self, *exc_info):
        return False

    def __getattr__(self, name):
        return getattr(self._reader, name)


def _browser_ua_feeds_for(uid: str) -> set[str]:
    now = time.monotonic()
    with _browser_ua_cache_lock:
        cached = _browser_ua_cache.get(uid)
        if cached is not None and (now - _browser_ua_cache_at.get(uid, 0.0)) < _BROWSER_UA_CACHE_TTL:
            return cached
    try:
        with tenancy.user_context(uid):
            with get_meta_connection() as conn:
                feeds = get_browser_ua_feed_urls(conn)
    except Exception:
        feeds = set()
    with _browser_ua_cache_lock:
        _browser_ua_cache[uid] = feeds
        _browser_ua_cache_at[uid] = now
    return feeds


def _invalidate_browser_ua_cache() -> None:
    with _browser_ua_cache_lock:
        _browser_ua_cache.clear()
        _browser_ua_cache_at.clear()


def _flag_browser_ua_on_refusal(feed_url: str) -> bool:
    """Flag a feed for browser-identity fetches after an honest-UA refusal and
    invalidate the cache so an immediate retry escalates. Returns True if newly
    flagged (a feed already flagged returns False, so the caller won't retry-loop)."""
    try:
        with get_meta_connection() as conn:
            newly = flag_browser_ua_feed(conn, feed_url, reason="refused honest UA on refresh")
    except Exception:
        return False
    if newly:
        _invalidate_browser_ua_cache()
        LOGGER.info("[refresh] flagged %s for browser-identity fetches", feed_url)
    return newly


def _proxy_feeds_for(uid: str) -> set[str]:
    now = time.monotonic()
    with _proxy_feeds_cache_lock:
        cached = _proxy_feeds_cache.get(uid)
        if cached is not None and (now - _proxy_feeds_cache_at.get(uid, 0.0)) < _PROXY_FEEDS_CACHE_TTL:
            return cached
    try:
        with tenancy.user_context(uid):
            with get_meta_connection() as conn:
                feeds = get_proxy_feed_urls(conn)
    except Exception:
        feeds = set()
    with _proxy_feeds_cache_lock:
        _proxy_feeds_cache[uid] = feeds
        _proxy_feeds_cache_at[uid] = now
    return feeds


def _invalidate_proxy_feeds_cache() -> None:
    with _proxy_feeds_cache_lock:
        _proxy_feeds_cache.clear()
        _proxy_feeds_cache_at.clear()


def _flag_proxy_feed_on_still_blocked(feed_url: str) -> bool:
    """Flag a feed for as-needed proxy escalation after browser-UA was already
    in play and the fetch still failed, and invalidate the cache so an
    immediate retry routes through the proxy. Mode-gated: only as_needed users
    ever get flagged here (off never wants it, always doesn't need per-feed
    tracking) — checked in the current (ambient) tenancy context, same as the
    scheduler's other per-user callbacks. Returns True if newly flagged."""
    if get_proxy_mode() != "as_needed":
        return False
    try:
        with get_meta_connection() as conn:
            newly = flag_proxy_feed(conn, feed_url, reason="still blocked after browser-UA")
    except Exception:
        return False
    if newly:
        _invalidate_proxy_feeds_cache()
        LOGGER.info("[refresh] flagged %s for as-needed proxy escalation", feed_url)
    return newly


def _tailscale_feeds_for(uid: str) -> set[str]:
    now = time.monotonic()
    with _tailscale_feeds_cache_lock:
        cached = _tailscale_feeds_cache.get(uid)
        if cached is not None and (now - _tailscale_feeds_cache_at.get(uid, 0.0)) < _TAILSCALE_FEEDS_CACHE_TTL:
            return cached
    try:
        with tenancy.user_context(uid):
            with get_meta_connection() as conn:
                feeds = get_tailscale_feed_urls(conn)
    except Exception:
        feeds = set()
    with _tailscale_feeds_cache_lock:
        _tailscale_feeds_cache[uid] = feeds
        _tailscale_feeds_cache_at[uid] = now
    return feeds


def _invalidate_tailscale_feeds_cache() -> None:
    with _tailscale_feeds_cache_lock:
        _tailscale_feeds_cache.clear()
        _tailscale_feeds_cache_at.clear()


def _flag_tailscale_feed_on_still_blocked(feed_url: str) -> bool:
    """Flag a feed for last-resort proxy escalation after the primary proxy was
    already in play and the fetch still failed, and invalidate the cache so an
    immediate retry routes through it. Mode-gated like _flag_proxy_feed_on_still_blocked,
    plus requires a last-resort backend actually configured — an unconfigured
    one is the opt-out, and a flag that could never take effect is just
    confusing clutter in Feed Properties. Returns True if newly flagged."""
    if get_proxy_mode() != "as_needed" or not get_tailscale_url():
        return False
    try:
        with get_meta_connection() as conn:
            newly = flag_tailscale_feed(conn, feed_url, reason="still blocked after proxy")
    except Exception:
        return False
    if newly:
        _invalidate_tailscale_feeds_cache()
        LOGGER.info("[refresh] flagged %s for last-resort proxy escalation", feed_url)
    return newly


def _flaresolverr_feeds_for(uid: str) -> set[str]:
    now = time.monotonic()
    with _flaresolverr_feeds_cache_lock:
        cached = _flaresolverr_feeds_cache.get(uid)
        if cached is not None and (now - _flaresolverr_feeds_cache_at.get(uid, 0.0)) < _FLARESOLVERR_FEEDS_CACHE_TTL:
            return cached
    try:
        with tenancy.user_context(uid):
            with get_meta_connection() as conn:
                feeds = get_flaresolverr_feed_urls(conn)
    except Exception:
        feeds = set()
    with _flaresolverr_feeds_cache_lock:
        _flaresolverr_feeds_cache[uid] = feeds
        _flaresolverr_feeds_cache_at[uid] = now
    return feeds


def _invalidate_flaresolverr_feeds_cache() -> None:
    with _flaresolverr_feeds_cache_lock:
        _flaresolverr_feeds_cache.clear()
        _flaresolverr_feeds_cache_at.clear()


def _flag_flaresolverr_feed_on_still_blocked(feed_url: str) -> bool:
    """Flag a feed for FlareSolverr escalation after the primary proxy was
    already in play and the fetch is STILL specifically a bot-challenge (see
    FeedRefreshService._is_bot_challenge — this callback is only ever invoked
    under that narrower gate, not any refusal). Mode-gated like the other two,
    plus requires FlareSolverr actually configured. Returns True if newly
    flagged."""
    if get_proxy_mode() != "as_needed" or not get_flaresolverr_url():
        return False
    try:
        with get_meta_connection() as conn:
            newly = flag_flaresolverr_feed(conn, feed_url, reason="still a bot challenge after proxy")
    except Exception:
        return False
    if newly:
        _invalidate_flaresolverr_feeds_cache()
        LOGGER.info("[refresh] flagged %s for FlareSolverr escalation", feed_url)
    return newly


def _proxy_is_down(uid: str) -> bool:
    with _proxy_down_lock:
        until = _proxy_down_until.get(uid)
    return until is not None and time.monotonic() < until


def _tailscale_is_down(uid: str) -> bool:
    with _tailscale_down_lock:
        until = _tailscale_down_until.get(uid)
    return until is not None and time.monotonic() < until


def _active_backend_for_fetch(uid: str, feed_url: str) -> tuple[str, str] | None:
    """Which outbound-proxy backend a fetch for feed_url would use for user uid,
    ignoring cooldowns — the precedence an attempt actually follows. Shared by
    _resolve_proxy_for_fetch (which then applies the cooldown gate) and
    _mark_backend_unreachable (diagnosing, after the fact, which backend just
    failed — the failure itself is what sets that backend's cooldown, so this
    intentionally does not consult it).

    Precedence follows how deep the escalation ladder goes: tailscale (last
    resort) outranks flaresolverr (a real challenge solve) which outranks the
    primary proxy — each is only ever flagged after the one before it was
    tried and found wanting. All three are reachable only in as_needed mode
    except the primary proxy's own "always" — never flaresolverr or tailscale
    in "always", which would spend a real Chrome instance (or the home IP) on
    every single fetch, the thing per-feed escalation exists to avoid."""
    mode = get_proxy_mode()
    if mode == "as_needed" and feed_url in _tailscale_feeds_for(uid):
        tailscale_url = get_tailscale_url()
        if tailscale_url:
            return ("tailscale", tailscale_url)
    if mode == "as_needed" and feed_url in _flaresolverr_feeds_for(uid):
        flaresolverr_url = get_flaresolverr_url()
        if flaresolverr_url:
            return ("flaresolverr", flaresolverr_url)
    if mode == "always":
        proxy_url = get_proxy_url()
        if proxy_url:
            return ("proxy", proxy_url)
    if mode == "as_needed" and feed_url in _proxy_feeds_for(uid):
        proxy_url = get_proxy_url()
        if proxy_url:
            return ("proxy", proxy_url)
    return None


def _mark_backend_unreachable(feed_url: str) -> None:
    uid = tenancy.current_user_id()
    backend = _active_backend_for_fetch(uid, feed_url)
    # Should always resolve (a fetch only gets proxied if _active_backend_for_fetch
    # said so in the first place) — "proxy" is a defensive fallback, not expected.
    name = backend[0] if backend else "proxy"
    if name == "flaresolverr":
        # Can't actually happen: this only fires on socks.ProxyError (see
        # FeedRefreshService._is_proxy_unreachable), and a FlareSolverr fetch
        # never touches pysocks at all — it's a plain HTTP POST to FlareSolverr's
        # own container, which internally handles any proxy stacking itself.
        # No-op rather than guessing which of the other two backends to blame.
        return
    if name == "tailscale":
        with _tailscale_down_lock:
            _tailscale_down_until[uid] = time.monotonic() + _TAILSCALE_DOWN_COOLDOWN_SECONDS
        LOGGER.warning("[refresh] tailscale unreachable for user %s — skipping it for %ds", uid, int(_TAILSCALE_DOWN_COOLDOWN_SECONDS))
    else:
        with _proxy_down_lock:
            _proxy_down_until[uid] = time.monotonic() + _PROXY_DOWN_COOLDOWN_SECONDS
        LOGGER.warning("[refresh] proxy unreachable for user %s — skipping it for %ds", uid, int(_PROXY_DOWN_COOLDOWN_SECONDS))


def _resolve_proxy_for_fetch(uid: str, feed_url: str) -> str | None:
    """Proxy URL to fetch feed_url through for user uid, or None for direct.

    "always" routes every fetch through the configured primary proxy.
    "as_needed" escalates progressively: feeds flagged by
    _flag_proxy_feed_on_still_blocked go through the primary proxy; feeds
    flagged one rung further by _flag_tailscale_feed_on_still_blocked (primary
    proxy already tried, still failing) go through the last-resort backend
    instead. "off" (the default) never proxies. Whichever backend would apply
    is skipped while IT is in its own unreachable cooldown — see
    _mark_backend_unreachable — falling through to direct rather than a
    tier that's currently down.

    Returns None for a feed FlareSolverr is actively handling instead — that
    tier is not a proxy swap at all (see _resolve_flaresolverr_for_fetch), and
    if this also set session.proxies for the same request, the request TO
    FlareSolverr's own container would get routed through the primary proxy
    by mistake, which is not what "stack FlareSolverr with the proxy" means
    (that stacking happens inside FlareSolverr's own request body instead)."""
    try:
        with tenancy.user_context(uid):
            backend = _active_backend_for_fetch(uid, feed_url)
            if backend is None:
                return None
            name, url = backend
            if name == "flaresolverr":
                return None
            if name == "tailscale":
                return None if _tailscale_is_down(uid) else url
            return None if _proxy_is_down(uid) else url
    except Exception:
        return None


# Moved to services/flaresolverr.py (shared with the page-fetch escalation
# ladder); kept as a name here since it's part of the module's tested surface.
_normalize_proxy_scheme_for_flaresolverr = flaresolverr_service.normalize_proxy_scheme


def _resolve_flaresolverr_for_fetch(uid: str, feed_url: str) -> tuple[str, str | None] | None:
    """(flaresolverr_endpoint, proxy_url_to_stack_or_None) for feed_url under
    user uid, or None to fetch it normally. The stack proxy is always the
    primary proxy when one is configured — confirmed empirically that a real
    Cloudflare-protected feed (cpp.libhunt.com) needed the stack: FlareSolverr
    alone, using the bare VPS IP, got a flat "error" (never even reached the
    challenge), while FlareSolverr + the primary proxy solved it cleanly."""
    try:
        with tenancy.user_context(uid):
            backend = _active_backend_for_fetch(uid, feed_url)
            if backend is None or backend[0] != "flaresolverr":
                return None
            proxy_url = get_proxy_url()
            stack = _normalize_proxy_scheme_for_flaresolverr(proxy_url) if proxy_url else None
            return (backend[1], stack)
    except Exception:
        return None


def get_reader():
    """Per-(thread, user) persistent Reader, resolved via the tenancy seam.

    The current user comes from :func:`tenancy.current_user_id` (defaults to the
    single legacy user). Each worker thread keeps an LRU pool of Reader handles
    keyed by user_id so cross-user requests on the same thread don't reopen, and
    so one user's handle is never shared across threads (SQLite affinity)."""
    uid = tenancy.current_user_id()
    pool = getattr(_reader_thread_local, "pool", None)
    if pool is None:
        pool = collections.OrderedDict()
        _reader_thread_local.pool = pool

    proxy = pool.get(uid)
    if proxy is not None:
        pool.move_to_end(uid)  # mark most-recently-used
        return proxy

    _open_start = time.perf_counter()
    proxy = _PersistentReaderProxy(
        ReaderApi(
            tenancy.reader_db_path(uid),
            browser_ua_provider=lambda u=uid: _browser_ua_feeds_for(u),
            proxy_resolver=lambda url, u=uid: _resolve_proxy_for_fetch(u, url),
            flaresolverr_resolver=lambda url, u=uid: _resolve_flaresolverr_for_fetch(u, url),
            session_timeout=(FEED_CONNECT_TIMEOUT_SECONDS, FEED_READ_TIMEOUT_SECONDS),
            entry_update_hook=lambda entry, is_new, u=uid: _full_content_entry_hook(u, entry, is_new),
        ).client()
    )
    pool[uid] = proxy
    _open_ms = int((time.perf_counter() - _open_start) * 1000)
    if _open_ms > 200:
        # A cold per-thread open is a suspect for multi-second folder switches (get_tagged_entry_keys
        # stalls with progress_steps=0, i.e. before any SQL runs); logged so that's confirmed or cleared.
        LOGGER.info("[perf] get_reader cold_open=%dms thread=%s", _open_ms, threading.current_thread().name)
    # Evict + close the least-recently-used handles beyond the per-thread cap.
    # Safe because the pool is thread-local and a thread serves one request at a
    # time, so an evicted handle is never mid-use on another thread.
    while len(pool) > _READER_POOL_MAX_PER_THREAD:
        _evicted_uid, evicted = pool.popitem(last=False)
        try:
            evicted._reader.close()
        except Exception:
            LOGGER.debug("evicted reader close failed", exc_info=True)
    return proxy


youtube_duration_service = YouTubeDurationService(
    get_durations_connection=get_yt_duration_connection,
    get_reader=get_reader,
    user_agent=READABILITY_USER_AGENT,
    # Per-user API key (with env fallback) so each user's key drives durations.
    api_key_provider=lambda: get_yt_api_key(),
    quota_sink=record_yt_quota_spend,
)

# Wire the quota-spend sink into the stateless YT modules so each billed API call
# (playlist list/insert, sub-sync channels/subscriptions) tallies against the
# current user's daily quota meter.
youtube_oauth_service.set_quota_sink(record_yt_quota_spend)
import services.youtube_sync as _youtube_sync_mod

_youtube_sync_mod.set_quota_sink(record_yt_quota_spend)

# Quire calls feed the per-user sliding-window rate meter (per-minute/hour).
quire_service.set_usage_sink(record_quire_call)

from services import feed_tags as feed_tags_service_mod
from services import reader_sanitize
from services.feed_tags import FeedTagService

# Feed-provided entry tags (<category>) captured at ingest: the sanitizing
# feed parser hands raw tag data to this service via the sink below.
feed_tag_service = FeedTagService(get_meta_connection=get_meta_connection)


def _store_feed_window(feed_url: str, entry_ids: list[str]) -> None:
    """Replace the recorded window (entry ids in the latest parse) for a feed.
    Called from the sanitizing parser inside reader's update, under the
    refreshing user's tenancy context."""
    try:
        with get_meta_connection() as conn:
            conn.execute("DELETE FROM feed_seen_window WHERE feed_url = ?", (feed_url,))
            conn.executemany(
                "INSERT OR IGNORE INTO feed_seen_window (feed_url, entry_id) VALUES (?, ?)",
                [(feed_url, eid) for eid in entry_ids],
            )
    except Exception:  # noqa: BLE001 — never fail a parse over window bookkeeping
        LOGGER.warning("[feed-window] store failed for %s", feed_url, exc_info=True)


reader_sanitize.set_entry_tag_sink(feed_tag_service.record_entry_tags)
reader_sanitize.set_feed_window_sink(_store_feed_window)
reader_sanitize.set_url_rewrite_provider(get_feed_url_rewrites)


def _page_fetch_backends() -> page_fetch.FetchBackends:
    """Backends available to the CURRENT tenancy for a single page fetch
    (tag/lead-image scraping, saved-article re-fetch) — resolved at fetch
    time so it reads the contextvars-scoped current user, same trick as the
    feed ladder's own per-user resolver lambdas (see _resolve_proxy_for_fetch
    just above). PageFetcher itself never touches settings or tenancy — it
    only sees whatever this returns, already resolved.

    Deliberately diverges from the feed ladder's mode gate: the feed ladder
    only reaches FlareSolverr/Tailscale in "as_needed" mode, because "always"
    would spend a real Chrome instance on every single feed poll. That
    reasoning doesn't apply here — this ladder is reactive by construction,
    only ever reaching FlareSolverr after the shallower tiers already failed
    on this exact URL — so a user on proxy_mode="always" still gets
    FlareSolverr rather than silently losing challenge-solving. See
    docs/architecture/feeds.md."""
    uid = tenancy.current_user_id()
    mode = get_proxy_mode()
    proxy = get_proxy_url() if (mode != "off" and not _proxy_is_down(uid)) else ""
    flare = get_flaresolverr_url() if mode != "off" else ""
    return page_fetch.FetchBackends(mode=mode, proxy_url=proxy, flaresolverr_url=flare)


# Shared by the tag/lead-image fetch (services/lead_images.py) and the
# saved-article re-fetch path (fetch_readability_article/fetch_full_page_article
# below) so a host FlareSolverr solved for one is immediately known to the
# other — see services/page_fetch.py and Plan.md.
page_fetcher = page_fetch.PageFetcher(
    backends=_page_fetch_backends,
    honest_user_agent=READABILITY_USER_AGENT,
)

lead_image_service = LeadImageService(
    get_meta_connection=get_meta_connection,
    get_reader=get_reader,
    user_agent=READABILITY_USER_AGENT,
    extract_video_id=youtube_duration_service.extract_video_id,
    page_fetcher=page_fetcher,
)


def _persist_page_tags(feed_url: str, entry_id: str, page_html: str, source_url: str | None = None) -> None:
    """Background source-HTML fetch sink: persist article-page tags for
    entries the feed never tagged. Stored as ``source='open'`` (not 'feed', so a feed refresh listing the entry untagged doesn't clear
    them; not 'page', which would auto-enable ingest-time page topics for the feed)."""
    if feed_tag_service.get_tags_for_entry(feed_url, entry_id):
        return
    tags = feed_tags_service_mod.extract_page_tags(page_html, source_url)
    if tags:
        feed_tag_service.record_entry_tags(feed_url, [(entry_id, tags)], source="open")


lead_image_service.set_page_tag_sink(_persist_page_tags)

# Lambda-wrapped so `sanitize_readability_html` resolves at call time (it's
# defined further down in this module).
starred_archive_service = StarredArchiveService(
    get_archive_connection=get_starred_archive_connection,
    get_meta_connection=get_meta_connection,
    get_reader=get_reader,
    user_agent=READABILITY_USER_AGENT,
    sanitize_readability_html=lambda html_text: sanitize_readability_html(html_text),
    # Lazy: _background_user_ids is defined later in this module; the worker
    # only calls this at runtime, so the name resolves by then.
    background_user_ids=lambda: _background_user_ids(),
    # Lazy for the same reason: rewrites redirector entry links (feedproxy/
    # feedburner) to the canonical URL the capture's source fetch resolved to.
    on_canonical_link=lambda f, e, old, new: _apply_canonical_entry_link(f, e, old, new),
    # Lazy for the same reason. Lets the star-restore backfill tell a
    # tag-created archive from a star-created one — since tag-as-keep, an
    # archive row alone no longer means the entry was starred.
    manually_tagged_keys=lambda: _manually_tagged_entry_keys(),
    # Lazy for the same reason, same purpose one axis over: Archive (done)
    # also keeps an entry's capture through an unstar (entry_has_keep_signal),
    # so an archive row alone doesn't mean "was starred" here either.
    archived_keys=lambda: get_archived_saved_keys(),
    # Lazy for the same reason. Per-feed policy (which extensions this feed
    # keeps) plus the link scan, so the service only has to ask one question.
    find_attachments=lambda feed_url, html, base: attachment_links_in_html(html, base, get_feed_attachment_exts(feed_url)),
    # Same per-feed policy, reused to gate enclosures (see the "4a" comment
    # in starred_archive.py) so one setting governs every non-image file
    # regardless of whether it was declared as an enclosure or found in the
    # body.
    attachment_allowed=lambda feed_url, url: _attachment_ext_matches(_url_ext(url), get_feed_attachment_exts(feed_url)),
)


def normalize_tag_value(value: str | None) -> str | None:
    """Normalize a tag, then fold it through any alias the user has defined."""
    normalized = normalize_tag_value_raw(value)
    return _apply_tag_alias(normalized) if normalized else normalized


def normalize_tag_value_raw(value: str | None) -> str | None:
    """Normalization only, with no alias applied.

    The alias editor needs this: normalizing "cpp" through the aliased path
    would return "c++" the moment the alias exists, so an alias could never be
    listed, edited or removed by the name it was created under.
    """
    if value is None:
        return None

    normalized = value.strip().lower().lstrip("#")
    # Collapse internal whitespace to hyphens so multi-word tags (e.g. imported
    # "games to play") survive as a single tag instead of being split or
    # rejected by TAG_VALUE_PATTERN, which does not allow spaces.
    normalized = re.sub(r"\s+", "-", normalized).strip("-")
    # Strip characters outside the tag alphabet instead of rejecting the whole
    # value — feed/page-provided tags like "AI & Machine Learning" or
    # "Tips/Tricks" must survive as ai-machine-learning / tipstricks rather
    # than silently vanishing. Collapse any hyphen runs that stripping leaves.
    normalized = re.sub(r"[^a-z0-9_.#+-]", "", normalized)
    normalized = re.sub(r"-{2,}", "-", normalized).strip("-")
    if not normalized:
        return None
    if not TAG_VALUE_PATTERN.fullmatch(normalized):
        return None
    return normalized


def get_tag_aliases() -> dict[str, str]:
    """``{alias: canonical}`` for the current user, cached."""
    try:
        uid = tenancy.current_user_id()
    except Exception:  # noqa: BLE001 — no tenancy (import time, a bare thread)
        return {}
    with _tag_alias_lock:
        cached = _tag_alias_cache.get(uid)
    if cached is not None:
        return cached
    try:
        with get_meta_connection() as conn:
            rows = conn.execute("SELECT alias, canonical FROM tag_aliases").fetchall()
        loaded = {str(r["alias"]): str(r["canonical"]) for r in rows}
    except Exception:  # noqa: BLE001 — a tagging path must not die over this
        LOGGER.debug("tag alias load failed", exc_info=True)
        return {}
    with _tag_alias_lock:
        _tag_alias_cache[uid] = loaded
    return loaded


def invalidate_tag_alias_cache(uid: str | None = None) -> None:
    with _tag_alias_lock:
        if uid is None:
            _tag_alias_cache.clear()
        else:
            _tag_alias_cache.pop(uid, None)


def _apply_tag_alias(normalized: str) -> str:
    """Fold an aliased spelling into its canonical tag.

    Exact match only. `c` (47 entries) sits right beside `cpp` and `c++` in the
    live vocabulary, so anything prefix- or fuzzy-matching would quietly file C
    posts under C++. One hop only, so a chain (or a cycle) cannot loop: an alias
    whose canonical is itself an alias resolves one step and stops.
    """
    return get_tag_aliases().get(normalized, normalized)


def normalize_search_query(value: str | None) -> str | None:
    if value is None:
        return None
    normalized = " ".join(value.strip().split())
    return normalized or None


def search_terms_from_query(value: str | None) -> list[str]:
    """Lowercased, whitespace-split search tokens — the shared tokenization
    every search surface (SQL-narrowed kept/feeds search, orphan-archive
    metadata match) filters on, so a query means the same thing everywhere."""
    normalized = normalize_search_query(value)
    return [token.lower() for token in normalized.split()] if normalized else []


def parse_manual_hashtags(raw_value: str | None) -> list[str]:
    if not raw_value:
        return []

    tags: list[str] = []
    seen: set[str] = set()
    for token in re.split(r"[\s,]+", raw_value):
        normalized = normalize_tag_value(token)
        if not normalized or normalized in seen:
            continue
        seen.add(normalized)
        tags.append(normalized)
        if len(tags) >= MAX_MANUAL_TAGS:
            break
    return tags


def _extract_tag_key(tag_record: object) -> str | None:
    if isinstance(tag_record, tuple) and tag_record:
        return str(tag_record[0])
    if isinstance(tag_record, str):
        return tag_record
    key = getattr(tag_record, "key", None)
    if key is None:
        return None
    return str(key)


def get_manual_tags_for_resource(reader, resource_id: tuple[str, str]) -> list[str]:
    tags: list[str] = []
    for tag_record in reader.get_tags(resource_id):
        key = _extract_tag_key(tag_record)
        if not key or not key.startswith(MANUAL_TAG_KEY_PREFIX):
            continue
        tag = key[len(MANUAL_TAG_KEY_PREFIX) :].strip().lower()
        if tag:
            tags.append(tag)
    return sorted(set(tags))


def _entry_is_starred(feed_url: str, entry_id: str) -> bool:
    """True if the entry has a saved_entries (star) row."""
    with get_meta_connection() as conn:
        return (
            conn.execute(
                "SELECT 1 FROM saved_entries WHERE feed_url = ? AND entry_id = ?",
                (feed_url, entry_id),
            ).fetchone()
            is not None
        )


def _entry_should_keep_archive(feed_url: str, entry_id: str) -> bool:
    """Whether an entry's offline archive must be retained.

    Since the tag-as-keep flip, both Star and Tag are keep signals: an archive is
    kept iff the entry is starred OR carries at least one manual tag. The star-off
    and last-tag-removal paths consult this before enqueuing an archive removal so
    dropping one axis never wipes an archive the other axis still needs.
    """
    if _entry_is_starred(feed_url, entry_id):
        return True
    return bool(get_manual_tags_for_entry(feed_url, entry_id))


def _get_orphan_manual_tags(feed_url: str, entry_id: str) -> list[str]:
    with get_meta_connection() as conn:
        rows = conn.execute(
            "SELECT tag FROM orphan_entry_tags WHERE feed_url = ? AND entry_id = ?",
            (feed_url, entry_id),
        ).fetchall()
    return sorted({str(r[0]) for r in rows})


def _set_orphan_manual_tags(feed_url: str, entry_id: str, next_tags: list[str]) -> bool:
    """Write the tag diff for an orphan-archive entry. Returns True if any tag
    was newly added (mirrors set_manual_tags_for_entry's added_any)."""
    with get_meta_connection() as conn:
        existing = {
            str(r[0])
            for r in conn.execute(
                "SELECT tag FROM orphan_entry_tags WHERE feed_url = ? AND entry_id = ?",
                (feed_url, entry_id),
            ).fetchall()
        }
        next_set = set(next_tags)
        for removed in existing - next_set:
            conn.execute(
                "DELETE FROM orphan_entry_tags WHERE feed_url = ? AND entry_id = ? AND tag = ?",
                (feed_url, entry_id, removed),
            )
        for added in next_set - existing:
            conn.execute(
                "INSERT OR IGNORE INTO orphan_entry_tags (feed_url, entry_id, tag) VALUES (?, ?, ?)",
                (feed_url, entry_id, added),
            )
        conn.commit()
        return bool(next_set - existing)


def set_manual_tags_for_entry(feed_url: str, entry_id: str, raw_tags: str | None) -> list[str]:
    next_tags = parse_manual_hashtags(raw_tags)

    added_any = False
    with get_reader() as reader:
        entry = reader.get_entry((feed_url, entry_id), None)
        if not entry:
            # Orphan archive: the feed is gone from `reader`, so there is no
            # resource for reader's own entry_tags to attach to. Star already
            # works this way (saved_entries is meta-DB, no reader dependency)
            # — give tags the same independence via orphan_entry_tags instead
            # of silently no-oping.
            added_any = _set_orphan_manual_tags(feed_url, entry_id, next_tags)
        else:
            resource_id = entry.resource_id
            existing_tags = get_manual_tags_for_resource(reader, resource_id)
            existing_set = set(existing_tags)
            next_set = set(next_tags)

            for removed in existing_set - next_set:
                reader.delete_tag(resource_id, f"{MANUAL_TAG_KEY_PREFIX}{removed}")

            for added in next_tags:
                if added in existing_set:
                    continue
                added_any = True
                # Use presence-only tag (no JSON value) to avoid type issues
                # with reader.set_tag's typed `value` parameter.
                reader.set_tag(resource_id, f"{MANUAL_TAG_KEY_PREFIX}{added}")

    invalidate_has_manual_tags_cache()
    invalidate_tag_counts_cache()

    # Tag is now a keep-forever signal: capture the whole entry offline when a tag
    # is added, and release the archive when the last tag is removed and the entry
    # isn't starred. Mirrors the star toggle's archive lifecycle.
    try:
        if added_any:
            starred_archive_service.enqueue_archive(feed_url, entry_id)
        elif not next_tags and not _entry_is_starred(feed_url, entry_id):
            starred_archive_service.enqueue_removal(feed_url, entry_id)
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("tag archive enqueue failed for %s/%s: %s", feed_url, entry_id, exc)

    return next_tags


def delete_manual_tag_everywhere(tag: str | None) -> int:
    """Strip a manual tag from every entry that carries it. Returns the number
    of entries the tag was removed from. Used by the tag-management delete
    action so a tag leaves the sidebar once nothing references it."""
    normalized = normalize_tag_value(tag)
    if not normalized:
        return 0

    key = f"{MANUAL_TAG_KEY_PREFIX}{normalized}"
    removed = 0
    failed = 0
    with get_reader() as reader:
        # reader filters entries by tag key in SQL, so this stays a single
        # pass across the whole library rather than a per-entry scan.
        for entry in list(reader.get_entries(tags=[key])):
            try:
                reader.delete_tag(entry.resource_id, key)
                removed += 1
                # Tag-as-keep: releasing the last tag on an unstarred entry
                # releases its offline archive too.
                feed_url, entry_id = entry.resource_id
                if not get_manual_tags_for_resource(reader, entry.resource_id) and not _entry_is_starred(feed_url, entry_id):
                    try:
                        starred_archive_service.enqueue_removal(feed_url, entry_id)
                    except Exception as exc:  # noqa: BLE001
                        LOGGER.warning("tag archive removal failed for %s/%s: %s", feed_url, entry_id, exc)
            except Exception:
                # Keep going so one bad entry doesn't abort the whole cleanup,
                # but log it so the failure is diagnosable.
                failed += 1
                LOGGER.warning(
                    "delete_manual_tag_everywhere: failed to remove %r from %s",
                    normalized,
                    entry.resource_id,
                    exc_info=True,
                )
    if failed:
        LOGGER.warning(
            "delete_manual_tag_everywhere: %r removed from %d entries, %d failed",
            normalized,
            removed,
            failed,
        )

    # Orphan archives keep this tag in orphan_entry_tags instead of reader's
    # entry_tags (see set_manual_tags_for_entry) — remove it there too, or
    # delete-everywhere would silently miss them and the tag would look gone
    # from the sidebar while still sitting on an orphan.
    with get_meta_connection() as conn:
        orphan_rows = conn.execute("SELECT feed_url, entry_id FROM orphan_entry_tags WHERE tag = ?", (normalized,)).fetchall()
        if orphan_rows:
            # One bulk delete rather than per-row: the per-row "any tags left?"
            # check below still needs a query per entry (it's asking about
            # OTHER tags), but removing this one doesn't.
            conn.execute("DELETE FROM orphan_entry_tags WHERE tag = ?", (normalized,))
            removed += len(orphan_rows)
            for feed_url, entry_id in orphan_rows:
                remaining = conn.execute(
                    "SELECT 1 FROM orphan_entry_tags WHERE feed_url = ? AND entry_id = ? LIMIT 1",
                    (feed_url, entry_id),
                ).fetchone()
                if not remaining and not _entry_is_starred(feed_url, entry_id):
                    try:
                        starred_archive_service.enqueue_removal(feed_url, entry_id)
                    except Exception as exc:  # noqa: BLE001
                        LOGGER.warning("tag archive removal failed for %s/%s: %s", feed_url, entry_id, exc)
        conn.commit()

    if removed:
        invalidate_has_manual_tags_cache()
        invalidate_tag_counts_cache()
    return removed


def clear_folder_curation(folder_id: int, remove_stars: bool, remove_tags: bool, only_tag: str | None = None) -> dict:
    """Bulk-remove stars and/or manual tags from every entry in a folder's feeds,
    so a whole folder drops out of the Saved/Kept view — without unsubscribing
    anything (deleting the folder would). The Saved-view counterpart to
    "move all visible", scoped to a folder so it is correct at any size.

    *only_tag* (with remove_tags) strips just that one tag rather than all manual
    tags — the "filter Saved by tag XYZ, remove XYZ from all shown" flow, scoped
    server-side so it isn't bounded by the client's paginated window.

    Releases the offline archive for any entry left with no curation, matching
    the single unstar/untag lifecycle."""
    result = {"stars_removed": 0, "tags_removed": 0, "entries_uncurated": 0}
    only_tag = normalize_tag_value(only_tag) if only_tag else None
    if not (remove_stars or remove_tags):
        return result
    with get_meta_connection() as conn:
        feeds = get_folder_feed_urls(conn, folder_id)
    if not feeds:
        return result

    affected: set[tuple[str, str]] = set()

    if remove_stars:
        with get_meta_connection() as conn:
            starred = [(str(f), str(i)) for f, i in conn.execute("SELECT feed_url, entry_id FROM saved_entries") if str(f) in feeds]
            for start in range(0, len(starred), 400):
                chunk = starred[start : start + 400]
                ph = ",".join("(?,?)" for _ in chunk)
                conn.execute(
                    f"DELETE FROM saved_entries WHERE (feed_url, entry_id) IN ({ph})",
                    [v for k in chunk for v in k],
                )
            conn.commit()
        result["stars_removed"] = len(starred)
        affected |= set(starred)

    if remove_tags:
        # Match a single tag key when only_tag is given, else any manual tag.
        match_sql = "key = ?" if only_tag else "key LIKE ?"
        match_val = f"{MANUAL_TAG_KEY_PREFIX}{only_tag}" if only_tag else f"{MANUAL_TAG_KEY_PREFIX}%"
        feed_list = list(feeds)
        tagged: set[tuple[str, str]] = set()
        conn = sqlite3.connect(str(tenancy.reader_db_path()), timeout=10.0)
        try:
            for i in range(0, len(feed_list), 900):
                fchunk = feed_list[i : i + 900]
                fph = ",".join("?" for _ in fchunk)
                for f, e in conn.execute(
                    f"SELECT DISTINCT feed, id FROM entry_tags WHERE {match_sql} AND feed IN ({fph})",
                    [match_val, *fchunk],
                ):
                    tagged.add((str(f), str(e)))
        finally:
            conn.close()
        with get_reader() as reader:
            for feed_url, entry_id in tagged:
                for t in reader.get_tags((feed_url, entry_id)):
                    key = t[0] if isinstance(t, tuple) else t
                    if not key or not key.startswith(MANUAL_TAG_KEY_PREFIX):
                        continue
                    # only_tag: strip just that one; else strip every manual tag.
                    if only_tag and key != f"{MANUAL_TAG_KEY_PREFIX}{only_tag}":
                        continue
                    try:
                        reader.delete_tag((feed_url, entry_id), key, missing_ok=True)
                        result["tags_removed"] += 1
                    except Exception:  # noqa: BLE001
                        LOGGER.warning("clear_folder_curation: untag failed %s", entry_id)
        affected |= tagged

    # Release archives for entries left with no curation (star or tag).
    for feed_url, entry_id in affected:
        try:
            if not _entry_is_starred(feed_url, entry_id) and not get_manual_tags_for_entry(feed_url, entry_id):
                starred_archive_service.enqueue_removal(feed_url, entry_id)
                result["entries_uncurated"] += 1
        except Exception:  # noqa: BLE001
            pass

    if result["stars_removed"] or result["tags_removed"]:
        invalidate_unread_counts_cache()
        invalidate_has_manual_tags_cache()
        invalidate_tag_counts_cache()
    return result


def tag_alias_preview(alias: str, canonical: str) -> dict:
    """What creating this alias would move, counted before anything is written.

    Both halves are reported because they live in different stores: manual tags
    are reader entry tags, captured publisher tags are rows in entry_feed_tags.
    An alias that only rewrote one of them would leave the split it exists to
    close.
    """
    alias_n = normalize_tag_value_raw(alias)
    canon_n = normalize_tag_value_raw(canonical)
    out = {"alias": alias_n, "canonical": canon_n, "manual": 0, "feed": 0, "error": None}
    if not alias_n or not canon_n:
        out["error"] = "Both a tag and its replacement are required."
        return out
    if alias_n == canon_n:
        out["error"] = "A tag cannot be an alias of itself."
        return out
    with get_reader() as reader:
        out["manual"] = reader.get_entry_counts(tags=[f"{MANUAL_TAG_KEY_PREFIX}{alias_n}"]).total
    with get_meta_connection() as conn:
        row = conn.execute("SELECT COUNT(*) AS n FROM entry_feed_tags WHERE tag = ?", (alias_n,)).fetchone()
        out["feed"] = int(row["n"]) if row else 0
        # A canonical that is itself aliased would make the new alias a two-hop
        # chain, and _apply_tag_alias deliberately only resolves one.
        # Chains are refused in BOTH directions, because _apply_tag_alias
        # deliberately takes one hop: a two-step chain would leave the first tag
        # resolving to something that no longer holds anything.
        chained = conn.execute("SELECT canonical FROM tag_aliases WHERE alias = ?", (canon_n,)).fetchone()
        if chained:
            out["error"] = f"#{canon_n} is itself an alias of #{chained['canonical']} — alias #{alias_n} to that instead."
            return out
        incoming = conn.execute("SELECT alias FROM tag_aliases WHERE canonical = ? LIMIT 1", (alias_n,)).fetchone()
        if incoming:
            out["error"] = (
                f"#{incoming['alias']} is already folded into #{alias_n}, so folding "
                f"#{alias_n} onward would strand it. Re-point #{incoming['alias']} first."
            )
    return out


def create_tag_alias(alias: str, canonical: str, *, rewrite: bool = True) -> dict:
    """Store an alias and (by default) fold the existing tags into it."""
    preview = tag_alias_preview(alias, canonical)
    if preview["error"]:
        return preview
    alias_n, canon_n = preview["alias"], preview["canonical"]
    with get_meta_connection() as conn:
        conn.execute(
            "INSERT INTO tag_aliases (alias, canonical, created_at) VALUES (?, ?, ?) "
            "ON CONFLICT(alias) DO UPDATE SET canonical = excluded.canonical",
            (alias_n, canon_n, time.time()),
        )
    invalidate_tag_alias_cache(tenancy.current_user_id())
    if rewrite:
        # Manual tags go through the existing rename, which already merges into
        # an existing tag rather than colliding with it.
        moved, _merged = rename_manual_tag_everywhere(alias_n, canon_n)
        preview["manual_moved"] = moved
        with get_meta_connection() as conn:
            # entry_feed_tags is (feed_url, entry_id, tag) — an entry that
            # already carries the canonical would collide, so drop those first.
            conn.execute(
                "DELETE FROM entry_feed_tags WHERE tag = ? AND EXISTS ("
                "  SELECT 1 FROM entry_feed_tags c WHERE c.tag = ?"
                "    AND c.feed_url = entry_feed_tags.feed_url"
                "    AND c.entry_id = entry_feed_tags.entry_id)",
                (alias_n, canon_n),
            )
            cur = conn.execute("UPDATE entry_feed_tags SET tag = ? WHERE tag = ?", (canon_n, alias_n))
            preview["feed_moved"] = cur.rowcount
    LOGGER.info("[tags] alias #%s -> #%s (manual=%s feed=%s)", alias_n, canon_n, preview.get("manual_moved"), preview.get("feed_moved"))
    return preview


def delete_tag_alias(alias: str) -> bool:
    """Drop an alias. Tags already folded stay folded — the rewrite happened."""
    alias_n = normalize_tag_value_raw(alias)
    if not alias_n:
        return False
    with get_meta_connection() as conn:
        cur = conn.execute("DELETE FROM tag_aliases WHERE alias = ?", (alias_n,))
    invalidate_tag_alias_cache(tenancy.current_user_id())
    return cur.rowcount > 0


def list_tag_aliases() -> list[dict]:
    with get_meta_connection() as conn:
        rows = conn.execute("SELECT alias, canonical, created_at FROM tag_aliases ORDER BY alias").fetchall()
    return [dict(r) for r in rows]


def rename_manual_tag_everywhere(old_tag: str | None, new_tag: str | None) -> tuple[int, bool]:
    """Rename a manual tag across every entry that carries it.

    Returns ``(count, merged)`` where *count* is the number of entries
    updated and *merged* is True if *new_tag* already had entries before
    the rename (i.e. the two tags were combined).
    """
    old_norm = normalize_tag_value(old_tag)
    new_norm = normalize_tag_value(new_tag)
    if not old_norm or not new_norm or old_norm == new_norm:
        return 0, False

    old_key = f"{MANUAL_TAG_KEY_PREFIX}{old_norm}"
    new_key = f"{MANUAL_TAG_KEY_PREFIX}{new_norm}"
    updated = 0
    with get_reader() as reader:
        merged = reader.get_entry_counts(tags=[new_key]).total > 0
        for entry in list(reader.get_entries(tags=[old_key])):
            try:
                reader.set_tag(entry.resource_id, new_key)
                reader.delete_tag(entry.resource_id, old_key)
                updated += 1
            except Exception:
                LOGGER.warning("rename_manual_tag_everywhere: failed on %s", entry.resource_id, exc_info=True)

    # Orphan archives keep tags in orphan_entry_tags instead of reader's
    # entry_tags (see set_manual_tags_for_entry) — rename there too.
    with get_meta_connection() as conn:
        if not merged:
            merged = conn.execute("SELECT 1 FROM orphan_entry_tags WHERE tag = ? LIMIT 1", (new_norm,)).fetchone() is not None
        orphan_rows = conn.execute("SELECT feed_url, entry_id FROM orphan_entry_tags WHERE tag = ?", (old_norm,)).fetchall()
        for feed_url, entry_id in orphan_rows:
            conn.execute(
                "DELETE FROM orphan_entry_tags WHERE feed_url = ? AND entry_id = ? AND tag = ?",
                (feed_url, entry_id, old_norm),
            )
            conn.execute(
                "INSERT OR IGNORE INTO orphan_entry_tags (feed_url, entry_id, tag) VALUES (?, ?, ?)",
                (feed_url, entry_id, new_norm),
            )
            updated += 1
        conn.commit()

    if updated:
        invalidate_has_manual_tags_cache()
        invalidate_tag_counts_cache()
    return updated, merged


def migrate_spaced_manual_tags() -> int:
    """One-time cleanup: manual tags imported with spaces (e.g. "games to play")
    were stored as a single key that the tag UI can't remove cleanly, since the
    remove/parse paths split on whitespace. Rewrite each spaced key to its
    hyphenated normalized form. Idempotent: once migrated no key contains a
    space, so the probing query returns nothing and this is a cheap no-op.

    Returns the number of (entry, tag) rewrites performed.
    """
    prefix = MANUAL_TAG_KEY_PREFIX
    try:
        conn = sqlite3.connect(str(tenancy.reader_db_path()), timeout=5.0)
        try:
            spaced_keys = [
                str(r[0])
                for r in conn.execute(
                    "SELECT DISTINCT key FROM entry_tags WHERE key LIKE ? AND key LIKE '% %'",
                    (f"{prefix}%",),
                ).fetchall()
            ]
        finally:
            conn.close()
    except Exception:
        LOGGER.warning("migrate_spaced_manual_tags: probe failed", exc_info=True)
        return 0

    if not spaced_keys:
        return 0

    rewrites = 0
    with get_reader() as reader:
        for old_key in spaced_keys:
            old_tag = old_key[len(prefix) :]
            new_norm = normalize_tag_value(old_tag)
            if not new_norm:
                continue
            new_key = f"{prefix}{new_norm}"
            if new_key == old_key:
                continue
            for entry in list(reader.get_entries(tags=[old_key])):
                try:
                    reader.set_tag(entry.resource_id, new_key)
                    reader.delete_tag(entry.resource_id, old_key)
                    rewrites += 1
                except Exception:
                    LOGGER.warning("migrate_spaced_manual_tags: failed on %s", entry.resource_id, exc_info=True)
    if rewrites:
        invalidate_has_manual_tags_cache()
        invalidate_tag_counts_cache()
        LOGGER.info("migrate_spaced_manual_tags: rewrote %d tag(s)", rewrites)
    return rewrites


def get_manual_tags_for_entry(feed_url: str, entry_id: str) -> list[str]:
    with get_reader() as reader:
        entry = reader.get_entry((feed_url, entry_id), None)
        if entry:
            return get_manual_tags_for_resource(reader, entry.resource_id)
    # Orphan archive: no reader resource, tags live in orphan_entry_tags instead.
    return _get_orphan_manual_tags(feed_url, entry_id)


def _manually_tagged_entry_keys() -> set[tuple[str, str]]:
    """Every (feed_url, entry_id) carrying a manual tag, in one query.

    Bulk because the caller (the starred-archive star-restore backfill) checks
    thousands of rows at startup; a per-row lookup there would be a reader round
    trip each time.
    """
    conn = sqlite3.connect(str(tenancy.reader_db_path()), timeout=10.0)
    try:
        return {
            (str(feed), str(entry_id))
            for feed, entry_id in conn.execute(
                "SELECT feed, id FROM entry_tags WHERE key LIKE ?",
                (f"{MANUAL_TAG_KEY_PREFIX}%",),
            )
        }
    finally:
        conn.close()


def get_feed_pinned_tags(feed_url: str) -> list[str]:
    """Tags the user pinned to this feed, to be offered on every one of its posts.

    Feed-provided tags only exist when the publisher ships them. A feed with a
    stable subject usually ships nothing — a guitar blog does not tag posts
    "guitar" because the whole site is about guitars — so filing them meant
    typing the same word every time. These are the answer to that, and they are
    a *suggestion*, never applied automatically: the point is one click, not an
    automatic tag (a rule already does that if you want it).
    """
    try:
        with get_meta_connection() as conn:
            row = conn.execute(
                "SELECT suggested_tags FROM feed_display_prefs WHERE feed_url = ?",
                (feed_url,),
            ).fetchone()
    except Exception:  # noqa: BLE001 — suggestions are never worth failing a render
        return []
    if not row or not row["suggested_tags"]:
        return []
    out: list[str] = []
    for raw in str(row["suggested_tags"]).replace(",", " ").split():
        normalized = normalize_tag_value(raw)
        if normalized and normalized not in out:
            out.append(normalized)
    return out


def set_feed_pinned_tags(feed_url: str, raw_value: str) -> list[str]:
    """Store the pinned tags for *feed_url*, normalized. Returns what was kept."""
    tags: list[str] = []
    for raw in (raw_value or "").replace(",", " ").split():
        normalized = normalize_tag_value(raw)
        if normalized and normalized not in tags:
            tags.append(normalized)
    with get_meta_connection() as conn:
        conn.execute(
            "INSERT INTO feed_display_prefs (feed_url, suggested_tags) VALUES (?, ?)"
            " ON CONFLICT(feed_url) DO UPDATE SET suggested_tags = excluded.suggested_tags",
            (feed_url, " ".join(tags)),
        )
    return tags


# Page extensions are never attachments, whatever the user types. A capture that
# followed .html/.php would stop being "keep the files this post links to" and
# become a crawler — and the one thing that reliably distinguishes a file from a
# page here IS the extension, so this list is what keeps the feature honest.
_NEVER_ATTACHMENT_EXTS = frozenset(
    {
        "htm",
        "html",
        "xhtml",
        "shtml",
        "php",
        "php3",
        "php4",
        "php5",
        "phtml",
        "asp",
        "aspx",
        "jsp",
        "jspx",
        "cgi",
        "pl",
        "cfm",
        "do",
        "action",
    }
)
# An exact extension ("gp5"), or a PREFIX pattern ("gp*" -> gp, gp3, gp4, gp5,
# gpx). Guitar Pro alone ships five, so listing them by hand is tedious and
# guaranteed to miss the next one.
#
# A bare "*" is still refused, and the prefix must be at least two characters:
# the point of the list is that it names a FAMILY of file types, and a one-letter
# prefix ("p*" -> pdf, png, ppt, psd…) is a wildcard wearing a hat. Page types
# are refused separately and at match time too, so even a broad prefix can never
# reach an .html.
_ATTACHMENT_EXT_RE = re.compile(r"^[a-z0-9]{1,8}$")
# What an auto-detected file extension must also match: at least one letter. A digits-only "extension" is a version or a number
# in a page URL (.../releases/tag/v3.1.0 -> "0"), not a file type. Explicit per-feed lists still accept digits.
_DETECTED_EXT_RE = re.compile(r"[a-z]")
_ATTACHMENT_PREFIX_RE = re.compile(r"^([a-z0-9]{2,8})\*$")
_HREF_ATTR_RE = re.compile(r'href=[\'"]([^\'"]+)[\'"]', re.I)


def _attachment_ext_matches(ext: str, patterns: list[str]) -> bool:
    """Whether a file's extension is one this feed keeps.

    Page types are re-checked HERE and not only on save: a stored value must
    never be able to smuggle one through, whatever wrote it.
    """
    if not ext or ext in _NEVER_ATTACHMENT_EXTS:
        return False
    for pattern in patterns:
        if pattern.endswith("*"):
            if ext.startswith(pattern[:-1]):
                return True
        elif ext == pattern:
            return True
    return False


# Per-file ceiling. A tab or a lyric sheet is kilobytes; anything past this is
# not what this feature is for, and the archive is a SQLite blob store.
ATTACHMENT_MAX_BYTES = 25 * 1024 * 1024


def normalize_attachment_exts(raw_value: str) -> list[str]:
    """Parse a user's extension list: lower-cased, dot-stripped, deduped.

    Page types are dropped rather than rejected — the user typing "pdf html"
    means the pdf, and failing the whole save over the html would be unhelpful.
    """
    out: list[str] = []
    for token in (raw_value or "").replace(",", " ").split():
        raw = token.strip().lower().lstrip(".")
        prefix_match = _ATTACHMENT_PREFIX_RE.match(raw)
        if prefix_match:
            # Kept as written ("gp*"). Page types are refused at MATCH time, so
            # a broad prefix still cannot reach an .html or a .php.
            value = prefix_match.group(1) + "*"
        elif _ATTACHMENT_EXT_RE.match(raw) and raw not in _NEVER_ATTACHMENT_EXTS:
            value = raw
        else:
            continue
        if value not in out:
            out.append(value)
    return out


# Already captured as images by the archive, so offering them as "attachments"
# would be redundant noise in the suggestion list.
# NB: bare extensions, and deliberately NOT named _IMAGE_EXTS — that name is
# already taken further down by a tuple of DOTTED suffixes for a different
# check, and being defined later it would win at import time.
_ARCHIVED_IMAGE_EXTS = frozenset(
    {
        "jpg",
        "jpeg",
        "png",
        "gif",
        "webp",
        "avif",
        "svg",
        "bmp",
        "ico",
    }
)
# A link to a bare domain ("https://example.com") leaves the TLD looking like a
# file extension. Nothing else distinguishes it from a real one.
_TLD_LOOKALIKES = frozenset(
    {
        "com",
        "org",
        "net",
        "io",
        "co",
        "us",
        "uk",
        "eu",
        "de",
        "fr",
        "jp",
        "ru",
        "tv",
        "me",
        "info",
        "biz",
        "app",
        "dev",
        "xyz",
        "social",
    }
)


def suppressed_attachment_exts(feed_url: str) -> set[str]:
    """Extension suggestions the user dismissed for this feed, lowercased for comparison."""
    try:
        with get_meta_connection() as conn:
            rows = conn.execute("SELECT ext FROM suppressed_feed_attachment_exts WHERE feed_url = ?", (feed_url,)).fetchall()
    except Exception:  # noqa: BLE001 — a suggestion is never worth an error
        LOGGER.debug("suppressed attachment ext lookup failed for %s", feed_url, exc_info=True)
        return set()
    return {str(r["ext"]).strip().lower() for r in rows}


def scan_feed_attachment_extensions(feed_url: str, limit: int = 10) -> list[dict]:
    """File extensions this feed's posts actually link to, most common first.

    The point is that the field stops guessing: a generic placeholder suggested
    Guitar Pro formats to every feed in the library. Counted from the entries
    already stored, so it costs no requests.

    Images are excluded (the archive captures those anyway), as are page types
    and the TLD lookalikes a bare-domain link leaves behind.
    """
    from collections import Counter

    dismissed = suppressed_attachment_exts(feed_url)
    counts: Counter[str] = Counter()
    try:
        with get_reader() as reader:
            for entry in reader.get_entries(feed=feed_url):
                content = entry.get_content(prefer_summary=False)
                html_text = (content.value if content and content.value else "") or ""
                if not html_text:
                    html_text = str(getattr(entry, "summary", "") or "")
                if not html_text:
                    continue
                for href in _HREF_ATTR_RE.findall(html_text):
                    path = urlparse(href).path.lower()
                    dot = path.rfind(".")
                    if dot < 0:
                        continue
                    ext = path[dot + 1 :]
                    if not _ATTACHMENT_EXT_RE.match(ext) or not _DETECTED_EXT_RE.search(ext):
                        continue
                    if ext in _NEVER_ATTACHMENT_EXTS or ext in _ARCHIVED_IMAGE_EXTS or ext in _TLD_LOOKALIKES or ext in dismissed:
                        continue
                    counts[ext] += 1
    except Exception:  # noqa: BLE001 — a suggestion is never worth an error
        LOGGER.debug("attachment extension scan failed for %s", feed_url, exc_info=True)
        return []
    # Drop one-offs: a path fragment that merely looks like an extension
    # ("…/music", "…/bulakhov") always appears once, while anything the feed
    # genuinely publishes recurs. This is a suggestion list, so a rare real type
    # costs nothing — it can still be typed.
    return [{"ext": ext, "count": n} for ext, n in counts.most_common(limit * 2) if n >= 2][:limit]


def get_feed_attachment_exts(feed_url: str) -> list[str]:
    """Extensions this feed keeps as attachments, or [] when the feature is off."""
    try:
        with get_meta_connection() as conn:
            row = conn.execute(
                "SELECT attachment_exts FROM feed_display_prefs WHERE feed_url = ?",
                (feed_url,),
            ).fetchone()
    except Exception:  # noqa: BLE001
        return []
    if not row or not row["attachment_exts"]:
        return []
    return normalize_attachment_exts(str(row["attachment_exts"]))


def set_feed_attachment_exts(feed_url: str, raw_value: str) -> list[str]:
    """Store the attachment extensions for *feed_url*. Returns what was kept."""
    exts = normalize_attachment_exts(raw_value)
    with get_meta_connection() as conn:
        conn.execute(
            "INSERT INTO feed_display_prefs (feed_url, attachment_exts) VALUES (?, ?)"
            " ON CONFLICT(feed_url) DO UPDATE SET attachment_exts = excluded.attachment_exts",
            (feed_url, " ".join(exts)),
        )
    return exts


# Attributes that carry a base64-encoded URL instead of an href. guitar-pro
# ships <span class="obflink" data-o="<base64>"> for its tab downloads, so the
# file the page offers every visitor is reachable by the browser but invisible
# to an href scan. Named attributes only — never a blind base64 sweep of the
# document, which would turn any long token into a candidate URL.
_OBFUSCATED_URL_ATTRS = ("data-o", "data-url", "data-href", "data-link", "data-file")
_B64ISH_RE = re.compile(r"^[A-Za-z0-9+/=_-]{16,}$")


def _decode_obfuscated_urls(soup) -> list[str]:
    """http(s) URLs hidden in base64 attributes, in document order.

    Only values that decode cleanly to an absolute http(s) URL are returned;
    anything else is ignored rather than guessed at. The result still has to
    satisfy the caller's extension list, so this widens where links are FOUND
    without widening what counts as a file.
    """
    out: list[str] = []
    for attr in _OBFUSCATED_URL_ATTRS:
        for el in soup.find_all(attrs={attr: True}):
            raw = str(el.get(attr) or "").strip()
            if not raw or not _B64ISH_RE.match(raw):
                continue
            try:
                decoded = base64.b64decode(raw + "=" * (-len(raw) % 4)).decode("utf-8")
            except Exception:  # noqa: BLE001 — not base64, or not text
                continue
            decoded = decoded.strip()
            if decoded.startswith(("http://", "https://")) and decoded not in out:
                out.append(decoded)
    return out


def attachment_links_in_html(content_html: str, base_url: str, exts: list[str]) -> list[str]:
    """Absolute URLs of linked files matching *exts*, in document order.

    Matches on the URL PATH only, so a query string never turns a page into an
    attachment (``/post.php?file=x.pdf`` is a page). Any host is allowed: the
    files a post links to routinely live on a separate asset domain — guitar-pro
    serves .gp tabs from assets-wp.guitar-pro.eu while the post is on
    blog.guitar-pro.com — so a same-host rule would miss exactly the case this
    exists for.
    """
    if not content_html or not exts:
        return []
    from bs4 import BeautifulSoup

    wanted = [e.lower() for e in exts]
    found: list[str] = []
    soup = BeautifulSoup(content_html, "html.parser")
    hrefs = [str(a.get("href") or "").strip() for a in soup.find_all("a")]
    # Links the page hides in a base64 attribute count too — see
    # _decode_obfuscated_urls.
    hrefs.extend(_decode_obfuscated_urls(soup))
    for href in hrefs:
        if not href or href.startswith(("data:", "mailto:", "javascript:")):
            continue
        absolute = urljoin(base_url or "", href)
        path = urlparse(absolute).path.lower()
        _dot = path.rfind(".")
        if _dot < 0:
            continue
        ext = path[_dot + 1 :]
        if _attachment_ext_matches(ext, wanted) and absolute not in found:
            found.append(absolute)
    return found


def candidate_attachment_links_in_html(content_html: str | None, base_url: str) -> list[str]:
    """Absolute URLs of every file-like link in *content_html*, regardless of
    any feed's attachment-extension policy.

    The per-entry Attachments panel's "available to save individually" list —
    a feed's chosen extensions govern what gets kept automatically, but a
    one-off file the feed's policy doesn't cover (or never will, because
    adding it broadly isn't worth it for one post) should still be reachable
    by hand. Same exclusions `scan_feed_attachment_extensions` already applies
    for its suggestion list (page types, images — the archive's own image scan
    already covers those, so listing them here would just be noise — and bare-
    domain TLD lookalikes), just not narrowed to one feed's list.
    """
    if not content_html:
        return []
    from bs4 import BeautifulSoup

    found: list[str] = []
    soup = BeautifulSoup(content_html, "html.parser")
    hrefs = [str(a.get("href") or "").strip() for a in soup.find_all("a")]
    hrefs.extend(_decode_obfuscated_urls(soup))
    for href in hrefs:
        if not href or href.startswith(("data:", "mailto:", "javascript:")):
            continue
        absolute = urljoin(base_url or "", href)
        path = urlparse(absolute).path.lower()
        dot = path.rfind(".")
        if dot < 0:
            continue
        ext = path[dot + 1 :]
        if not _ATTACHMENT_EXT_RE.match(ext) or not _DETECTED_EXT_RE.search(ext):
            continue
        if ext in _NEVER_ATTACHMENT_EXTS or ext in _ARCHIVED_IMAGE_EXTS or ext in _TLD_LOOKALIKES:
            continue
        if absolute not in found:
            found.append(absolute)
    return found


FEED_TAGS_SUPPRESS_ALL = "*"


def _chip_tags_for_rows(keys: list[tuple[str, str]]) -> dict[tuple[str, str], list[str]]:
    """Normalized publisher tags per row, as the entry pane would show them as chips.

    Same dismissal rules as ``get_feed_tag_suggestions`` (per-feed, global, "*" for all), batched for a list. Tags a feed
    pins in Feed Properties are not read at all, so they only count when the publisher also ships them.
    """
    raw = feed_tag_service.get_tags_for_entries(keys)
    if not raw:
        return {}
    global_dismissed = {n for d in feed_tag_service.global_suppressed_tags() if (n := normalize_tag_value(d))}
    dismissed_by_feed: dict[str, set[str] | None] = {}
    out: dict[tuple[str, str], list[str]] = {}
    for key, tags in raw.items():
        feed_url = key[0]
        if feed_url not in dismissed_by_feed:
            dismissed = feed_tag_service.suppressed_tags(feed_url)
            dismissed_by_feed[feed_url] = (
                None
                if any((d or "").strip() == FEED_TAGS_SUPPRESS_ALL for d in dismissed)
                else {n for d in dismissed if (n := normalize_tag_value(d))}
            )
        per_feed = dismissed_by_feed[feed_url]
        if per_feed is None:
            continue
        shown: list[str] = []
        for tag in tags:
            norm = normalize_tag_value(tag)
            if norm and norm not in per_feed and norm not in global_dismissed and norm not in shown:
                shown.append(norm)
        shown = shown[:MAX_FEED_TAG_SUGGESTIONS]  # dismiss first, then cap -- same order as get_feed_tag_suggestions
        if shown:
            out[key] = shown
    return out


def get_feed_tag_suggestions(feed_url: str, entry_id: str) -> list[str]:
    """Feed-provided tags captured at ingest (entry_feed_tags meta table).

    **Nothing is filtered automatically; dismissal is the user's, per (feed, tag).**

    Two heuristics were tried and both hid tags the user wanted:

    1. *Coverage* — suppress a tag on ~every entry, on the theory that it carries
       no per-entry signal. Killed by a guitarplayer.com tag feed: "Lessons" is on
       every post AND is exactly the right tag when filing a guitar lesson. These
       chips are for **filing**, not for telling entries apart, so uniformity is
       not disqualifying at all.
    2. *Feed-name echo* — suppress a uniform tag that restates the feed's title.
       Killed by the live title, "Latest from Guitar Player in Lessons": a tag
       feed puts its tag in its own name, so "Lessons" echoes it too. Matching the
       feed URL fails identically — `/r/VinylDeals/` and `/feeds/tag/lessons` have
       the same shape.

    The difference between "VinylDeals" (a place, useless) and "Lessons" (a kind of
    content, wanted) is semantic, and no signal in the feed metadata expresses it.
    **Showing a useless chip is cheap — it is ignored. Hiding a wanted one is
    invisible**, so everything is shown until the user says otherwise. The × on a
    chip records that decision in `suppressed_feed_tags`, undoable from Feed
    Properties. Resist a third heuristic — the first two each looked convincing
    against the data that motivated them.

    A tag value can also be dismissed globally (`suppressed_feed_tags_global`,
    editable from Settings -> Tags) — this is a *different* axis, not a stronger
    version of the per-feed one: it is for a tag value that is never worth
    filing under on any feed (e.g. "comments"), decided once instead of
    per-feed whack-a-mole, and still just filters the suggestion chip, not a
    rule that acts on entries carrying the tag.
    """
    try:
        tags = feed_tag_service.get_tags_for_entry(feed_url, entry_id)
        dismissed = feed_tag_service.suppressed_tags(feed_url)
        global_dismissed = feed_tag_service.global_suppressed_tags()
    except Exception:
        LOGGER.warning("feed tag suggestion lookup failed for %s", feed_url, exc_info=True)
        return []
    # Compare through normalize_tag_value on BOTH sides. The chips are rendered
    # normalized (lowercased, spaces to hyphens), so the × sends "popular-deals"
    # while the stored feed tag is "Popular Deals" — a plain lowercase compare
    # gives "popular deals" and misses. Single-word tags normalize to themselves,
    # which is why "python" and "accu" stuck and every multi-word tag came back.
    # "*" means this feed shows no suggestions at all. Dismissing tag by tag is
    # whack-a-mole on a feed whose tags are never wanted — a webcomic tagged
    # "Comics" and "Cartoons" says nothing about the strip, and each new tag the
    # publisher adds needs dismissing again. Stored in the same table, so it
    # undoes from the same place (Feed Properties -> Hidden tags).
    if any((d or "").strip() == FEED_TAGS_SUPPRESS_ALL for d in dismissed):
        return []
    dismissed_norm = {normalize_tag_value(d) for d in dismissed} | {normalize_tag_value(d) for d in global_dismissed}
    return [t for t in tags if normalize_tag_value(t) not in dismissed_norm][:MAX_FEED_TAG_SUGGESTIONS]


def has_any_manual_tags() -> bool:
    """Single fast SQL probe of reader's entry_tags table to check whether
    *any* entry has a manual tag. Lets us skip the per-entry tag scan entirely
    for the common case (no manual tags). Cached for a minute; mutations can
    invalidate via invalidate_has_manual_tags_cache()."""
    now = time.time()
    with _has_manual_tags_lock:
        cached = _has_manual_tags_cache.get("any")
        if cached and now - cached[0] < HAS_MANUAL_TAGS_CACHE_TTL_SECONDS:
            return cached[1]
    try:
        conn = sqlite3.connect(str(tenancy.reader_db_path()), timeout=2.0)
        try:
            row = conn.execute(
                "SELECT 1 FROM entry_tags WHERE key LIKE ? LIMIT 1",
                (f"{MANUAL_TAG_KEY_PREFIX}%",),
            ).fetchone()
            present = row is not None
        finally:
            conn.close()
        if not present:
            # Orphan archives keep tags in orphan_entry_tags (meta DB) instead
            # of reader's entry_tags — check there too, or an orphan-only tag
            # would make this probe (and everything gated on it) miss it.
            with get_meta_connection() as meta_conn:
                present = meta_conn.execute("SELECT 1 FROM orphan_entry_tags LIMIT 1").fetchone() is not None
    except Exception:
        present = True  # safe default — fall back to the slow path
    with _has_manual_tags_lock:
        _has_manual_tags_cache["any"] = (time.time(), present)
    return present


def invalidate_has_manual_tags_cache() -> None:
    with _has_manual_tags_lock:
        _has_manual_tags_cache.clear()


# The library-wide tagged-key set, cached briefly: the home route asks for it twice per request (unread badge and
# per-folder counts) and the full read is ~17k rows. Shares _has_manual_tags_cache, so every tag write that
# invalidates that also drops this; the short TTL covers write paths that don't (imports, merges).
_TAGGED_KEYS_CACHE_TTL_SECONDS = 15


def get_tagged_entry_keys(feed_urls: set[str] | None = None) -> set[tuple[str, str]]:
    """(feed_url, entry_id) of every manually-tagged entry — the "tag" half of the
    Kept view's star-OR-tag set. When *feed_urls* is given, restrict to those feeds
    (chunked to stay under SQLite's variable limit); None scans the whole library.

    Reads reader's ``entry_tags`` directly: it lives in reader's DB (not the meta
    DB) and reader's high-level API has no bulk tag-key scan. Goes through
    ``get_reader()``'s pooled connection (10s busy_timeout, wal_autocheckpoint
    pragmas already applied by ``_LectioReaderStorage.setup_db``) rather than
    opening a fresh ``sqlite3.connect`` per call — suspected 2026-09-01 as the
    real cost behind the badges gap_block stall surviving the
    get_saved_unread_count batching fix (see Plan.md's refresh-contention item):
    a raw connection pays file-open/schema-load/mmap-setup on every call, and
    under refresh writing to the same WAL file that cost compounds. Logged when
    slow so a live stall pins the blame here or clears it. Shared by the Kept
    view list, per-folder counts, and the sidebar badge so the three stay
    consistent."""
    with _has_manual_tags_lock:
        cached = _has_manual_tags_cache.get("tagged_keys")
    db_path = str(tenancy.reader_db_path())
    if cached and cached[2] == db_path and time.time() - cached[0] < _TAGGED_KEYS_CACHE_TTL_SECONDS:
        all_keys = cached[1]
        return set(all_keys) if feed_urls is None else {k for k in all_keys if k[0] in feed_urls}
    if feed_urls is not None:
        # Fill the cache from the full read, then narrow: one scan serves every scope.
        all_keys = get_tagged_entry_keys(None)
        return {k for k in all_keys if k[0] in feed_urls}
    keys: set[tuple[str, str]] = set()
    like = f"{MANUAL_TAG_KEY_PREFIX}%"
    _start = time.perf_counter()
    # Refresh-contention investigation (Plan.md Tier 1, Read Above/Below lead):
    # this query's own SLOW_SQL logging never fires, even when this function
    # measures 1-2s+ -- db.execute() for a plain SELECT only prepares the
    # statement and does not eagerly fetch, so the real scan happens lazily
    # during the `for row in ...:` iteration below, outside the wrapped
    # execute() call's timing. _lectio_progress_steps (set by _TimedConnection)
    # is a connection-level counter that keeps accumulating through that
    # iteration though, since the progress handler fires on every VDBE step
    # regardless of which Python call triggered it -- reading it right after
    # each chunk's loop finishes (before the next chunk's execute() resets it)
    # gives an accurate total, answering directly whether a slow call here is
    # genuine scan cost or SQLite lock-wait.
    _progress_steps = 0
    try:
        with get_reader() as reader:
            db = reader._storage.get_db()
            for row in db.execute("SELECT feed, id FROM entry_tags WHERE key LIKE ?", (like,)):
                keys.add((str(row[0]), str(row[1])))
            _progress_steps += getattr(db, "_lectio_progress_steps", 0)
        with _has_manual_tags_lock:
            _has_manual_tags_cache["tagged_keys"] = (time.time(), keys, db_path)
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("get_tagged_entry_keys failed: %s", exc)
    elapsed_ms = int((time.perf_counter() - _start) * 1000)
    if elapsed_ms > 200:
        LOGGER.info(
            "[perf] get_tagged_entry_keys=%dms rows=%d progress_steps=%d",
            elapsed_ms,
            len(keys),
            _progress_steps,
        )
    return set(keys)


def get_entry_keys_for_manual_tag(feed_urls: set[str], tag: str) -> set[tuple[str, str]]:
    """(feed_url, entry_id) of entries carrying manual tag *tag*, in the view's
    feeds. Lets the Saved+tag view narrow to the tag in SQL before hydrating,
    instead of hydrating the whole kept set and post-filtering in Python — the
    latter fetched lead images for thousands of entries to render a tag view
    that might match none (measured at ~38s for an empty `inbox` view)."""
    keys: set[tuple[str, str]] = set()
    key = f"{MANUAL_TAG_KEY_PREFIX}{tag}"
    if not feed_urls:
        return keys
    try:
        conn = sqlite3.connect(str(tenancy.reader_db_path()), timeout=5.0)
        try:
            feed_list = list(feed_urls)
            for i in range(0, len(feed_list), 900):
                chunk = feed_list[i : i + 900]
                ph = ",".join("?" for _ in chunk)
                for row in conn.execute(
                    f"SELECT feed, id FROM entry_tags WHERE key = ? AND feed IN ({ph})",
                    [key, *chunk],
                ):
                    keys.add((str(row[0]), str(row[1])))
        finally:
            conn.close()
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("get_entry_keys_for_manual_tag failed: %s", exc)
    return keys


def get_entry_ids_with_feed_tag(feed_urls: set[str], tag: str) -> set[tuple[str, str]]:
    """(feed_url, entry_id) of entries carrying feed-provided tag *tag*
    (entry_feed_tags, the publisher-shipped suggestions — not a manual tag),
    restricted to *feed_urls*. Backs the entry pane's click-to-preview on a
    suggested-tag chip: unlike the tag_filter automation rule, this is a
    one-off read-path filter over whatever the current view already fetched,
    not a persisted rule, so it lives in the meta DB (entry_feed_tags) and
    normalizes at read time the same way the tag_filter matcher does (raw
    stored values aren't pre-normalized)."""
    keys: set[tuple[str, str]] = set()
    if not feed_urls:
        return keys
    normalized = normalize_tag_value(tag)
    if not normalized:
        return keys
    feed_list = list(feed_urls)
    with get_meta_connection() as conn:
        for i in range(0, len(feed_list), 900):
            chunk = feed_list[i : i + 900]
            ph = ",".join("?" for _ in chunk)
            for row in conn.execute(
                f"SELECT feed_url, entry_id, tag FROM entry_feed_tags WHERE feed_url IN ({ph})",
                chunk,
            ):
                if normalize_tag_value(str(row["tag"])) == normalized:
                    keys.add((str(row["feed_url"]), str(row["entry_id"])))
    return keys


def invalidate_tag_counts_cache() -> None:
    with tag_counts_cache_lock:
        tag_counts_cache.clear()


def get_tag_counts_for_feeds(feed_urls: set[str]) -> list[dict[str, int | str]]:
    if not feed_urls:
        return []

    # Fast path: zero manual tags exist anywhere → skip entirely.
    if not has_any_manual_tags():
        return []

    key = tuple(sorted(feed_urls))
    now = time.time()
    with tag_counts_cache_lock:
        cached = tag_counts_cache.get(key)
        if cached:
            ts, value = cached
            if now - ts < TAG_COUNTS_CACHE_TTL_SECONDS:
                return value

    # Single bulk query instead of per-entry reader.get_tags() calls.
    prefix = MANUAL_TAG_KEY_PREFIX
    sorted_feeds = sorted(feed_urls)
    placeholders = ",".join("?" * len(sorted_feeds))
    try:
        conn = sqlite3.connect(str(tenancy.reader_db_path()), timeout=5.0)
        try:
            # COUNT(DISTINCT id), not COUNT(*): the same article syndicated
            # across two feeds (same entry id under two feed URLs) is one row per
            # feed in entry_tags, but the post list collapses it to a single item
            # via build_entry_dedupe_key. Counting distinct ids keeps the sidebar
            # tally consistent with what clicking the tag actually shows.
            rows = conn.execute(
                f"SELECT key, COUNT(DISTINCT id) FROM entry_tags WHERE key LIKE ? AND feed IN ({placeholders}) GROUP BY key",
                [f"{prefix}%", *sorted_feeds],
            ).fetchall()
        finally:
            conn.close()
    except Exception:
        rows = []

    counts: dict[str, int] = {}
    for raw_key, count in rows:
        tag = raw_key[len(prefix) :].strip().lower()
        if tag:
            counts[tag] = counts.get(tag, 0) + count

    result = [{"name": tag, "count": counts[tag]} for tag in sorted(counts)]
    with tag_counts_cache_lock:
        tag_counts_cache[key] = (now, result)
    return result


def get_all_manual_tag_names() -> list[str]:
    """Every distinct manual-tag name in the library, sorted — the source for
    tag-input autocomplete. A cheap DISTINCT over the indexed key column; empty
    when no manual tags exist at all (the common fast path)."""
    if not has_any_manual_tags():
        return []
    prefix = MANUAL_TAG_KEY_PREFIX
    names: set[str] = set()
    try:
        conn = sqlite3.connect(str(tenancy.reader_db_path()), timeout=5.0)
        try:
            rows = conn.execute("SELECT DISTINCT key FROM entry_tags WHERE key LIKE ?", [f"{prefix}%"]).fetchall()
        finally:
            conn.close()
        names.update(name for (k,) in rows if (name := str(k)[len(prefix) :].strip().lower()))
    except Exception:  # noqa: BLE001 — autocomplete is a nicety, never fail the page
        pass
    # Orphan-only tags (see set_manual_tags_for_entry) still belong in the
    # autocomplete list — otherwise a tag used only on orphans is untypeable
    # via suggestion.
    try:
        with get_meta_connection() as meta_conn:
            names.update(str(r[0]) for r in meta_conn.execute("SELECT DISTINCT tag FROM orphan_entry_tags").fetchall())
    except Exception:  # noqa: BLE001
        pass
    return sorted(names)


def get_favicon_url(feed_url: str, site_url: str | None = None) -> str | None:
    url_for_host = site_url or feed_url
    host = urlparse(url_for_host).hostname
    if not host:
        host = urlparse(feed_url).hostname
    if not host:
        return None
    return f"/api/favicon?domain={quote_plus(host)}"


def feed_display_title(feed, fallback: str = "") -> str:
    """The name to show for a feed: the user's rename if there is one, else the feed's own title.

    ``reader`` keeps three fields — ``title`` (what the feed calls itself), ``user_title`` (the rename) and
    ``resolved_title`` (user_title or title). Reading ``title`` directly ignores renames, which is what made
    a renamed feed keep its old name on every post while the sidebar showed the new one. One helper so the
    display paths cannot drift apart again.
    """
    return str(getattr(feed, "resolved_title", None) or getattr(feed, "title", None) or fallback)


def _feed_url_display_host(url: str) -> str:
    """Bare host of a feed URL for disambiguation text — no userinfo/port, leading www. folded."""
    try:
        net = urlparse(url).netloc.split("@")[-1].split(":")[0].lower()
    except Exception:
        return ""
    return net[4:] if net.startswith("www.") else net


def _disambiguate_feed_titles(feeds: "list[FeedInFolder]") -> None:
    """Two feeds sharing one folder can display the identical title — a
    scraped/renamed feed, or two publishers who both called their feed
    "Latest News" — and the only thing that tells them apart today is the
    URL in the row's hover tooltip, easy to miss and no help at all on
    touch. Appends " — host" to every feed in a same-title group so the
    sidebar row itself shows the difference. Falls back to the full URL for
    a group that also shares a host (two feed variants on the same site) --
    a duplicate-everything collision is rare enough that a plain longer
    suffix beats inventing a second disambiguator. Mutates titles in place,
    intended to run once per folder's already-sorted feed list.
    """
    by_title: dict[str, list[FeedInFolder]] = {}
    for f in feeds:
        by_title.setdefault(f.title, []).append(f)
    for group in by_title.values():
        if len(group) < 2:
            continue
        hosts = [_feed_url_display_host(f.url) for f in group]
        host_disambiguates = len(hosts) == len(set(hosts)) and all(hosts)
        for f, host in zip(group, hosts, strict=True):
            f.title = f"{f.title} — {host}" if host_disambiguates else f"{f.title} — {f.url}"


def get_feed_title_map() -> dict[str, str]:
    now = time.time()
    with feed_title_map_cache_lock:
        cached = feed_title_map_cache.get("titles")
        if cached and now - cached[0] < FEED_TITLE_MAP_CACHE_TTL_SECONDS:
            return dict(cached[1])
    titles: dict[str, str] = {}
    with get_reader() as reader:
        for feed in reader.get_feeds():
            titles[feed.url] = feed.resolved_title or feed.title or feed.url
    with feed_title_map_cache_lock:
        feed_title_map_cache["titles"] = (time.time(), dict(titles))
    return titles


def get_feed_fetch_history(conn, feed_url: str, limit: int = 30) -> list[dict]:
    """Recent per-refresh fetch attempts for a feed, newest first, for the
    Feed Properties → History tab."""
    rows = conn.execute(
        "SELECT fetched_at, status, http_status, new_entries, duration_ms, error"
        " FROM feed_fetch_history WHERE feed_url = ? ORDER BY fetched_at DESC LIMIT ?",
        (feed_url, limit),
    ).fetchall()
    history: list[dict] = []
    for r in rows:
        raw_at = r["fetched_at"]
        try:
            fetched_at = format_datetime_for_ui(datetime.fromtimestamp(float(raw_at), tz=timezone.utc)) if raw_at else None
        except Exception:
            fetched_at = None
        history.append(
            {
                "fetched_at": fetched_at,
                "status": str(r["status"] or ""),
                "http_status": r["http_status"],
                "new_entries": r["new_entries"],
                "duration_ms": r["duration_ms"],
                "error": r["error"],
            }
        )
    return history


_AUTOMATION_TYPE_LABELS = {
    "highlight": "Highlight",
    "mark_as_read": "Auto mark read",
    "deduplicate": "Deduplicate",
    "email_article": "Email article",
    "webhook": "Webhook",
    "youtube_playlist": "Add to YT playlist",
    "instapaper": "Save to Instapaper",
    "save_article": "Save/Star article",
    "quire": "Add to Quire",
    "tag_filter": "Tag filter",
}
_AUTOMATION_SCOPE_LABELS = {"global": "All feeds", "folder": "Folder", "folders": "Folders", "feed": "This feed", "feeds": "Selected feeds"}


def _automation_rule_detail(rule: dict) -> str:
    """A short human description of what a rule does, per rule type."""
    rule_type = str(rule.get("type", ""))
    search_in = str(rule.get("search_in") or "title")
    if rule_type == "highlight":
        return f"{rule.get('color') or 'yellow'} · matches {search_in}"
    if rule_type == "mark_as_read":
        return f"matches {search_in}"
    if rule_type == "tag_filter":
        return str(rule.get("keyword") or "").strip() or "feed tags"
    if rule_type == "deduplicate":
        method = str(rule.get("keyword") or "slug")
        window = int(rule.get("dedup_window_hours") or 24)
        return f"{method} match · {window}h window"
    if rule_type == "email_article":
        to = str(rule.get("email_to") or "").strip()
        return f"to {to}" if to else "email"
    if rule_type == "webhook":
        fmt = str(rule.get("webhook_format") or "generic")
        return f"POST · {fmt} · matches {search_in}"
    if rule_type == "instapaper":
        return f"save · matches {search_in}"
    if rule_type == "save_article":
        return f"star + inbox tag · matches {search_in}"
    if rule_type == "youtube_playlist":
        parts = ["→ " + (str(rule.get("yt_playlist_title") or "") or "playlist")]
        kw = str(rule.get("keyword") or "").strip()
        parts.append(f"keyword “{kw}”" if kw else "all videos")
        if rule.get("yt_include_shorts"):
            parts.append("incl. Shorts")
        lo = int(rule.get("yt_min_minutes") or 0)
        hi = int(rule.get("yt_max_minutes") or 0)
        if lo or hi:
            parts.append(f"{lo}–{hi}m" if hi else f"≥{lo}m")
        if rule.get("yt_mark_read"):
            parts.append("mark read")
        return " · ".join(parts)
    return ""


def collect_feed_automations(conn, feed_url: str, folder_ids: list[int]) -> dict:
    """Automations that act on this feed: configured rules in scope + recent runs.

    Rules come from the highlight_keywords table (which holds every rule type, not
    just highlights); a rule applies to this feed when its scope is global, this
    feed directly, or a folder the feed belongs to. Recent runs come from
    rule_run_log_entries, the per-entry record of what automation actually fired
    on this feed's entries — so the user can see what Lectio is doing without
    reading code."""
    folder_id_set = {int(f) for f in folder_ids}
    rules: list[dict] = []
    for rule in get_highlight_keywords(conn):
        scope = str(rule.get("scope", ""))
        scope_id = str(rule.get("scope_id") or "")
        if scope == "global":
            applies = True
        elif scope == "feed":
            applies = scope_id == feed_url
        elif scope == "feeds":
            applies = feed_url in parse_feeds_scope_id(scope_id)
        elif scope == "folder":
            applies = scope_id.isdigit() and int(scope_id) in folder_id_set
        elif scope == "folders":
            applies = bool(folder_id_set & set(parse_folders_scope_id(scope_id)))
        else:
            applies = False
        if not applies:
            continue
        rule_type = str(rule.get("type", ""))
        rules.append(
            {
                "type": rule_type,
                "type_label": _AUTOMATION_TYPE_LABELS.get(rule_type, rule_type or "Rule"),
                "scope_label": _AUTOMATION_SCOPE_LABELS.get(scope, scope),
                "keyword": str(rule.get("keyword") or ""),
                "enabled": bool(rule.get("enabled")),
                "detail": _automation_rule_detail(rule),
            }
        )

    recent_runs: list[dict] = []
    run_rows = conn.execute(
        "SELECT l.run_at AS run_at, l.rule_type AS rule_type, l.keyword AS keyword,"
        " l.trigger AS trigger, COUNT(e.entry_id) AS affected"
        " FROM rule_run_log l JOIN rule_run_log_entries e ON e.log_id = l.id"
        " WHERE e.feed_url = ?"
        " GROUP BY l.id ORDER BY l.run_at DESC LIMIT 15",
        (feed_url,),
    ).fetchall()
    for r in run_rows:
        run_at_raw = r["run_at"]
        try:
            run_at = format_datetime_for_ui(datetime.fromisoformat(str(run_at_raw)))
        except Exception:
            run_at = str(run_at_raw or "")
        rule_type = str(r["rule_type"] or "")
        recent_runs.append(
            {
                "run_at": run_at,
                "type_label": _AUTOMATION_TYPE_LABELS.get(rule_type, rule_type or "Rule"),
                "keyword": str(r["keyword"] or ""),
                "trigger": str(r["trigger"] or ""),
                "affected": int(r["affected"] or 0),
            }
        )

    return {"rules": rules, "recent_runs": recent_runs}


def get_feed_properties(feed_url: str) -> dict:
    with get_reader() as reader:
        feed_obj = reader.get_feed(feed_url, None)

        if feed_obj is None:
            # Not in reader — either unsubscribed (its saves/tags survive in the
            # starred archive, kept-stars-unsub's whole point) or a genuinely
            # unknown URL. Only the former gets a minimal, editable response:
            # real_title + suggested_tags (both meta-DB-only, no reader needed)
            # so a tagged orphan batch can still offer its suggestion chip —
            # everything else the modal fetches (health, counts, folder, image
            # strategy, etc.) has no live-feed meaning and stays absent; the
            # client already defaults every one of those fields on a falsy value.
            archived_title = starred_archive_service.get_orphan_feed_title(feed_url)
            if archived_title is None:
                return {
                    "feed_url": feed_url,
                    "found": False,
                    "error": "Feed not found.",
                }
            return {
                "feed_url": feed_url,
                "found": True,
                "is_orphan": True,
                "real_title": archived_title or feed_url,
                "health": "unsubscribed",
                "health_detail": "This feed was unsubscribed — showing archived saves only.",
                "suggested_tags": get_feed_pinned_tags(feed_url),
            }

        total_posts = 0
        unread_posts = 0
        newest_post_dt = None
        last_received_dt = None
        for entry in reader.get_entries(feed=feed_url):
            total_posts += 1
            if not entry.read:
                unread_posts += 1

            published_dt = entry.published or entry.updated
            if published_dt and (newest_post_dt is None or published_dt > newest_post_dt):
                newest_post_dt = published_dt

            if entry.added and (last_received_dt is None or entry.added > last_received_dt):
                last_received_dt = entry.added

        last_exception = getattr(feed_obj, "last_exception", None)
        if last_exception:
            health = "error"
            health_detail = feed_refresh_service.humanize_feed_exception(last_exception)
        elif getattr(feed_obj, "updates_enabled", True):
            health = "ok"
            health_detail = "No known errors."
        else:
            health = "paused"
            health_detail = "Updates are disabled for this feed."

        img_strategy, _, img_strategy_manual = lead_image_service.get_feed_strategy(feed_url)
        _feed_domain = urlparse(feed_url).netloc.lower()
        with get_meta_connection() as _pc:
            _disp = get_feed_display_prefs(_pc, feed_url)
            _folder_full_content = full_content_fetch.folder_enabled(_pc, feed_url)
            _folder_page_topics = page_topics.folder_enabled(_pc, feed_url)
            _strat_rows = _pc.execute(
                "SELECT strategy, image_url, fetched_at, error, image_alt, image_title "
                "FROM feed_strategy_cache WHERE feed_url = ? ORDER BY strategy",
                (feed_url,),
            ).fetchall()
            _feed_backoff_row = _pc.execute(
                "SELECT consecutive_failures, next_retry_at FROM feed_failure_state WHERE feed_url = ?",
                (feed_url,),
            ).fetchone()
            _domain_backoff_row = (
                _pc.execute(
                    "SELECT consecutive_failures, next_retry_at FROM domain_failure_state WHERE domain = ?",
                    (_feed_domain,),
                ).fetchone()
                if _feed_domain
                else None
            )
        _strat_cache = [
            {
                "strategy": r["strategy"],
                "image_url": r["image_url"],
                "fetched_at": format_datetime_for_ui(datetime.fromtimestamp(r["fetched_at"], tz=timezone.utc)) if r["fetched_at"] else None,
                "error": r["error"],
                "image_alt": r["image_alt"],
                "image_title": r["image_title"],
            }
            for r in _strat_rows
        ]
        _folder_id_rows = _pc.execute(
            "SELECT DISTINCT folder_id FROM folder_feeds WHERE feed_url = ?",
            (feed_url,),
        ).fetchall()
        _devto_id = devto_service.devto_feed_id_from_url(feed_url)

        _now_ts = time.time()
        _feed_next_retry = float(_feed_backoff_row["next_retry_at"]) if _feed_backoff_row and _feed_backoff_row["next_retry_at"] else None
        _domain_next_retry = (
            float(_domain_backoff_row["next_retry_at"]) if _domain_backoff_row and _domain_backoff_row["next_retry_at"] else None
        )
        _feed_failures = int(_feed_backoff_row["consecutive_failures"]) if _feed_backoff_row else 0
        _domain_failures = int(_domain_backoff_row["consecutive_failures"]) if _domain_backoff_row else 0
        _effective_next_retry = max(f for f in [_feed_next_retry or 0.0, _domain_next_retry or 0.0]) or None
        _backoff_active = bool(_effective_next_retry and _effective_next_retry > _now_ts)
        _backoff_domain_driven = _backoff_active and (_domain_next_retry or 0.0) > (_feed_next_retry or 0.0)

        # If the reader reports no active error but our failure counter is still
        # non-zero, surface a warning so the user isn't confused by "No known errors"
        # next to a failure count.  This happens when the feed last fetched
        # successfully (clearing last_exception) but the failure counter hasn't been
        # reset yet (e.g. the feed was in backoff when the last success occurred).
        if health == "ok" and _feed_failures > 0:
            health = "warning"
            _nf = _feed_failures
            health_detail = (
                f"Last update succeeded, but {_nf} consecutive failure"
                f"{'s' if _nf != 1 else ''} still on record. "
                "Will auto-clear on next successful refresh."
            )

        props = {
            "feed_url": feed_url,
            "found": True,
            "title": getattr(feed_obj, "resolved_title", None) or getattr(feed_obj, "title", None) or feed_url,
            "user_title": getattr(feed_obj, "user_title", None),
            "real_title": getattr(feed_obj, "title", None),
            "website": _rewrite_url_host(getattr(feed_obj, "link", None), get_dedupe_host_aliases()),
            "added": format_datetime_for_ui(getattr(feed_obj, "added", None)),
            "last_updated": format_datetime_for_ui(getattr(feed_obj, "last_updated", None)),
            "last_received": format_datetime_for_ui(last_received_dt),
            "last_post": format_datetime_for_ui(newest_post_dt),
            "total_posts": total_posts,
            "unread_posts": unread_posts,
            "updates_enabled": bool(getattr(feed_obj, "updates_enabled", True)),
            "health": health,
            "health_detail": health_detail,
            "image_strategy": img_strategy if img_strategy_manual else "auto",
            "image_strategy_detected": img_strategy,
            "show_lead_image_in_article": bool(_disp.get("show_lead_image_in_article", 1)),
            "show_lead_image_as_thumb": bool(_disp.get("show_lead_image_as_thumb", 1)),
            "show_image_caption": int(_disp.get("show_image_caption", -1)),
            "caption_source": _disp.get("caption_source") or "auto",
            "hide_shorts": bool(_disp.get("hide_shorts", 0)),
            "hide_unpremiered": bool(_disp.get("hide_unpremiered", 0)),
            "hide_members_only": bool(_disp.get("hide_members_only", 0)),
            "hide_locked_comics": bool(_disp.get("hide_locked_comics", 0)),
            "hide_paywalled": bool(_disp.get("hide_paywalled", 0)),
            "auto_read_days": int(_disp.get("auto_read_days", 0)),
            "auto_read_cap": auto_read_parent_cap(_pc, feed_url=feed_url),
            "inject_source_images": bool(_disp.get("inject_source_images", 0)),
            "katex_dollar_math": bool(_disp.get("katex_dollar_math", 0)),
            "fetch_full_content": int(_disp.get("fetch_full_content", full_content_fetch.FEED_INHERIT)),
            "folder_fetch_full_content": _folder_full_content,
            "capture_page_topics": int(_disp.get("capture_page_topics", full_content_fetch.FEED_INHERIT)),
            "folder_capture_page_topics": _folder_page_topics,
            "feed_thumbnail_url": _disp.get("feed_thumbnail_url") or None,
            "thumb_crop": str(_disp.get("thumb_crop") or "cover"),
            "thumb_strategy": _disp.get("thumb_strategy") or None,
            "smart_min_scale": float(_disp["smart_min_scale"]) if _disp.get("smart_min_scale") is not None else None,
            "fill_zoom": float(_disp["fill_zoom"]) if _disp.get("fill_zoom") is not None else None,
            "is_youtube_feed": "youtube.com/feeds/videos.xml" in feed_url,
            "devto_feed_id": _devto_id,
            "devto": devto_service.get_feed_config(_pc, _devto_id) if _devto_id else None,
            "browser_ua": feed_url in get_browser_ua_feed_urls(_pc),
            "proxied": feed_url in get_proxy_feed_urls(_pc),
            "tailscaled": feed_url in get_tailscale_feed_urls(_pc),
            "flaresolverred": feed_url in get_flaresolverr_feed_urls(_pc),
            # Declared domain migrations for this feed. Rendered as the "Other
            # domains" list under Website, which is the only way to see them —
            # they otherwise act invisibly at ingest and in the global dedupe
            # alias map.
            "url_rewrites": [{"from_host": f, "to_host": t} for f, t in get_feed_url_rewrites(feed_url)],
            # Dismissed suggestion chips, so a mis-clicked × has a way back.
            "suppressed_tags": feed_tag_service.suppressed_tag_list(feed_url),
            "suggested_tags": get_feed_pinned_tags(feed_url),
            "attachment_exts": get_feed_attachment_exts(feed_url),
            "strategy_cache": _strat_cache,
            "folder_ids": [int(r["folder_id"]) for r in _folder_id_rows],
            "fetch_history": get_feed_fetch_history(_pc, feed_url),
            "automations": collect_feed_automations(_pc, feed_url, [int(r["folder_id"]) for r in _folder_id_rows]),
            "backoff_active": _backoff_active,
            "backoff_domain_driven": _backoff_domain_driven,
            "backoff_domain": _feed_domain if _backoff_domain_driven else None,
            "backoff_retry_at": (
                format_datetime_for_ui(datetime.fromtimestamp(_effective_next_retry, tz=timezone.utc)) if _effective_next_retry else None
            ),
            "backoff_feed_failures": _feed_failures,
            "backoff_domain_failures": _domain_failures,
        }

        # If the stored title looks like a URL (often when a feed was just added),
        # try a short metadata fetch of the feed to get a human-friendly title.
        try:
            current_title = str(props.get("title") or "")
            if current_title == feed_url or current_title.startswith("http"):
                try:
                    # SSRF-guarded: feed_url comes straight from the query string,
                    # so validate it (and every redirect hop) before fetching.
                    with url_guard.build_client(timeout=3.0) as _tc:
                        resp = url_guard.safe_get(_tc, feed_url)
                    if resp.status_code == 200 and resp.content:
                        parsed = feedparser.parse(resp.content)
                        fetched_title = None
                        try:
                            feed_section = getattr(parsed, "feed", None)
                            if isinstance(feed_section, dict):
                                fetched_title = feed_section.get("title")
                        except Exception:
                            fetched_title = None

                        if not fetched_title:
                            try:
                                # parsed may be a FeedParserDict-like object; cast to dict
                                parsed_dict = dict(parsed)
                                feed_obj = parsed_dict.get("feed")
                                if isinstance(feed_obj, dict):
                                    fetched_title = feed_obj.get("title")
                            except Exception:
                                fetched_title = None

                        if fetched_title:
                            fetched_title = html.unescape(str(fetched_title).strip())
                            if fetched_title and fetched_title != current_title:
                                props["title"] = fetched_title
                                if not props.get("real_title"):
                                    props["real_title"] = fetched_title
                except Exception:
                    # Don't fail the request for a metadata fetch error.
                    pass
        except Exception:
            pass

        # If there are no posts yet, and updates are enabled and there is no
        # known feed error, start a background update so the UI can show posts
        # shortly after the request. We don't block the request on the update.
        try:
            should_queue_update = total_posts == 0 and health != "error" and bool(getattr(feed_obj, "updates_enabled", True))
        except Exception:
            should_queue_update = False

        if should_queue_update:
            with updating_feeds_lock:
                already_updating = feed_url in updating_feeds
            if not already_updating:
                _start_background_update(feed_url)
                props["update_queued"] = True
            else:
                props["update_queued"] = True

        return props


def get_folder_properties(folder_id: int) -> dict:
    with get_meta_connection() as conn:
        folder_row = conn.execute(
            """
            SELECT f.id, f.name, f.cadence_minutes, f.retention_days, f.auto_read_days, f.fetch_full_content, f.capture_page_topics,
                CASE WHEN f.parent_id IS NULL THEN f.name
                     ELSE root.name || ' / ' || f.name END AS path
            FROM folders f
            LEFT JOIN folders root ON f.parent_id = root.id
            WHERE f.id = ?
            """,
            (folder_id,),
        ).fetchone()

        if not folder_row:
            return {"found": False, "error": "Folder not found."}

        feed_urls = get_folder_feed_urls(conn, folder_id)
        feed_count = len(feed_urls)
        # Deleted = tombstoned posts (retention/purge/manual delete). Tombstones
        # age out with the tomb-sweeping window, so this reads as "deleted
        # recently", which is the useful number.
        deleted_articles = 0
        if feed_urls:
            _ph = ",".join("?" * len(feed_urls))
            deleted_articles = int(
                conn.execute(
                    f"SELECT COUNT(*) FROM deleted_entries WHERE feed_url IN ({_ph})",
                    sorted(feed_urls),
                ).fetchone()[0]
            )

    if not feed_urls:
        return {
            "found": True,
            "folder_id": folder_id,
            "name": folder_row["name"],
            "path": folder_row["path"],
            "cadence_minutes": folder_row["cadence_minutes"],
            "retention_days": folder_row["retention_days"],
            "auto_read_days": folder_row["auto_read_days"],
            "auto_read_cap": auto_read_parent_cap(),
            "fetch_full_content": bool(folder_row["fetch_full_content"]),
            "capture_page_topics": bool(folder_row["capture_page_topics"]),
            "deleted_articles": 0,
            "feed_count": 0,
            "total_articles": 0,
            "unread_articles": 0,
            "top_feeds": [],
        }

    total_articles = 0
    unread_articles = 0
    feed_stats: dict[str, dict] = {}

    # Aggregate in SQL instead of hydrating every entry. This used to loop
    # `reader.get_entries(feed=url)` and count in Python, which on the Deals
    # folder — 17 feeds, 31,843 entries — made Folder Properties take **74
    # seconds**. Nothing on this dialog needs an Entry object: only a count, an
    # unread count, and the oldest date per feed. (`newest` was computed the
    # same expensive way and never read.)
    #
    # `coalesce(published, first_updated)` is the expression the entry sort
    # window already uses, for the same reason: an entry with no published date
    # should fall back to when reader first saw it rather than sort as NULL.
    # The trade is that `oldest` no longer honours per-entry date overrides,
    # and it feeds only the articles-per-week estimate below.
    def _as_utc(raw) -> datetime | None:
        if not raw:
            return None
        try:
            dt = datetime.fromisoformat(str(raw))
        except ValueError:
            return None
        return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)

    urls = sorted(feed_urls)
    with get_reader() as reader:
        db = reader._storage.get_db()
        # Chunked: the root folder resolves to every feed in the library, well
        # past SQLite's bound-variable limit.
        for start in range(0, len(urls), 900):
            chunk = urls[start : start + 900]
            placeholders = ",".join("?" * len(chunk))
            rows = db.execute(
                f"""
                SELECT feed, COUNT(*), SUM(CASE WHEN read THEN 0 ELSE 1 END),
                       MIN(COALESCE(published, first_updated))
                FROM entries WHERE feed IN ({placeholders}) GROUP BY feed
                """,
                chunk,
            ).fetchall()
            for feed_url_row, count, unread, oldest in rows:
                count = int(count or 0)
                total_articles += count
                unread_articles += int(unread or 0)
                feed_stats[str(feed_url_row)] = {
                    "title": None,
                    "count": count,
                    "oldest": _as_utc(oldest),
                }

        for url in list(feed_stats):
            feed_obj = reader.get_feed(url, None)
            if feed_obj:
                feed_stats[url]["title"] = getattr(feed_obj, "resolved_title", None) or getattr(feed_obj, "title", None) or url

    now = datetime.now(tz=timezone.utc)
    top_feeds = []
    for url, fs in feed_stats.items():
        count = fs["count"]
        oldest = fs["oldest"]
        try:
            if oldest and count > 0:
                # Span from oldest article to today; floor at 1 week so
                # bulk-imported or very new feeds don't produce absurd numbers.
                span_weeks = max((now - oldest).total_seconds() / (7 * 86400), 1.0)
                avg_per_week = round(count / span_weeks, 1)
            else:
                avg_per_week = 0.0
        except Exception:
            avg_per_week = 0.0
        top_feeds.append(
            {
                "feed_url": url,
                "title": fs["title"] or url,
                "avg_per_week": avg_per_week,
                "total": count,
            }
        )

    top_feeds.sort(key=lambda x: x["avg_per_week"], reverse=True)

    return {
        "found": True,
        "folder_id": folder_id,
        "name": folder_row["name"],
        "path": folder_row["path"],
        "cadence_minutes": folder_row["cadence_minutes"],
        "retention_days": folder_row["retention_days"],
        "auto_read_days": folder_row["auto_read_days"],
        "auto_read_cap": auto_read_parent_cap(),
        "fetch_full_content": bool(folder_row["fetch_full_content"]),
        "capture_page_topics": bool(folder_row["capture_page_topics"]),
        "deleted_articles": deleted_articles,
        "feed_count": feed_count,
        "total_articles": total_articles,
        "unread_articles": unread_articles,
        "top_feeds": top_feeds[:8],
    }


def format_datetime_for_ui(dt: datetime | None) -> str | None:
    if dt is None:
        return None
    # A handful of feeds publish dates that are technically valid datetimes but
    # outside the range Python can convert to local time (e.g. year 1 or far
    # future). astimezone() raises OverflowError on those. Fall back to a
    # tz-naive render rather than 500'ing the whole page on one bad entry.
    if dt.tzinfo is not None:
        try:
            dt = dt.astimezone()
        except OverflowError, ValueError:
            dt = dt.replace(tzinfo=None)
    hour = dt.hour % 12 or 12
    minute = f":{dt.minute:02d}"
    am_pm = "am" if dt.hour < 12 else "pm"
    return f"{dt.strftime('%b')} {dt.day}, {dt.year} {hour}{minute}{am_pm}"


def datetime_sort_value(dt: datetime | None) -> float:
    if dt is None:
        return float("-inf")
    try:
        if dt.tzinfo is not None:
            return dt.timestamp()
        return dt.replace(tzinfo=None).timestamp()
    except Exception:
        return float("-inf")


def _format_size_bytes(n: int) -> str:
    """Human-readable byte count for the Saved/Kept size column. No Jinja
    filter for this exists in this app's template environment (only
    ``urlencode`` is registered), and every other display value here is
    formatted in Python before it reaches the template, not in it."""
    size = float(n)
    for unit in ("B", "KB", "MB", "GB"):
        if size < 1024 or unit == "GB":
            return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
        size /= 1024
    return f"{size:.1f} GB"


# Many feeds (web comics in particular) omit <pubDate> but encode the date in
# the entry URL path. Sort breaks badly without it — every undated entry maps
# to -inf and clusters at one end. This regex catches the common /YYYY/MM/DD/
# pattern; we use it only as a sort fallback so display still shows what the
# feed actually said (or '—').
# Both common dated-permalink shapes: /2019/07/06/slug (WordPress) and
# /2025-11-22/slug (Jekyll/Hugo and hand-rolled blogs — brendangregg.com uses
# it). Only the first separator has to be a slash; the rest may be either, so
# one pattern covers both without a second regex to keep in sync.
_URL_PUBDATE_RE = re.compile(r"/(\d{4})[/-](\d{1,2})[/-](\d{1,2})(?:/|$|\?|\.)")

# WordPress's other common permalink shape: /YYYY/MM/slug, no day. blog.guitar-pro.com
# uses it for all 67 of its undated entries. Month precision beats no date at all —
# these otherwise sit at the Unix epoch and sort to one end — and the month itself is
# trustworthy, since a WordPress permalink is generated from the publish date.
#
# Deliberately NOT filled in from the page: that site publishes `dateModified` and
# nothing else ("Last update: oct. 21, 2024" on a post whose URL says 2021/04), so
# reading a date off the page would be wrong by three and a half years. A real
# month beats a precise-looking lie.
_URL_PUBMONTH_RE = re.compile(r"/(\d{4})/(\d{1,2})/(?:$|[^0-9])")

_MONTHNAMES = (
    "january",
    "february",
    "march",
    "april",
    "may",
    "june",
    "july",
    "august",
    "september",
    "october",
    "november",
    "december",
)

# datagenetics.com's permalink shape: /blog/march112020/, /blog/august42012/ —
# month name + a number + year, run together with no separator. That number is
# NOT the day: the blog's own archive index (blog.html) lists posts within a
# "March 2020" section in descending-number order, and march112020 sits well
# after the feed's own <pubDate> for march102020 (2020-03-27) — it's a per-month
# post sequence, not a day-of-month. datagenetics ships no <pubDate> at all for
# ~35 older posts and no date anywhere on the article page either.
#
# The number is still used as the returned day (clamped to <=28, so it is
# always a valid date) rather than always the 1st: real same-month pubDates
# climb monotonically with it (march1..march10 above run March 4 -> March 27),
# so a higher number reliably sorts later within the month even though it is
# not the true day. A distinct, wrong-but-ordered day beats every post in a
# month colliding on the 1st.
_URL_MONTHNAME_YEAR_RE = re.compile(r"/(" + "|".join(_MONTHNAMES) + r")(\d{1,2})((?:19|20)\d{2})/", re.I)


def url_inferred_pubdate(link: str | None) -> datetime | None:
    if not link:
        return None
    match = _URL_PUBDATE_RE.search(link)
    if not match:
        return None
    try:
        year, month, day = int(match.group(1)), int(match.group(2)), int(match.group(3))
    except ValueError:
        return None
    if not (2000 <= year <= 2099 and 1 <= month <= 12 and 1 <= day <= 31):
        return None
    try:
        return datetime(year, month, day, tzinfo=timezone.utc)
    except ValueError, OverflowError:
        return None


def url_inferred_pubmonth(link: str | None) -> datetime | None:
    """The /YYYY/MM/ or month-name+year permalink shape, resolved to that month.

    Month precision — the day is not a claim, only an ordering hint where one is
    available. See ``_URL_PUBMONTH_RE`` and ``_URL_MONTHNAME_YEAR_RE`` for why
    this is still the best signal those posts have, and for why the month-name
    shape's day comes from its counter rather than always being the 1st.
    """
    if not link:
        return None
    day = 1
    match = _URL_PUBMONTH_RE.search(link)
    if match:
        year, month = int(match.group(1)), int(match.group(2))
    else:
        match = _URL_MONTHNAME_YEAR_RE.search(link)
        if not match:
            return None
        month = _MONTHNAMES.index(match.group(1).lower()) + 1
        # Clamped to 1..28: the regex's \d{1,2} accepts 0 (e.g. "march02020"
        # backtracks to sequence "0"), and datetime() raises on day=0 rather
        # than being a plausibility check this function can catch below.
        day = max(1, min(int(match.group(2)), 28))
        year = int(match.group(3))
    if not (2000 <= year <= 2099 and 1 <= month <= 12):
        return None
    return datetime(year, month, day, tzinfo=timezone.utc)


# Some feeds prefix entry titles with the date ("2024-01-15: …", "2024/01/15
# …"). Same idea as URL-based inference — useful as a sort fallback. ISO-ish
# only; ambiguous mm/dd/yyyy vs dd/mm/yyyy patterns intentionally ignored.
_TITLE_PUBDATE_RE = re.compile(r"^\s*(\d{4})[-/.](\d{1,2})[-/.](\d{1,2})\b")


def title_inferred_pubdate(title: str | None) -> datetime | None:
    if not title:
        return None
    match = _TITLE_PUBDATE_RE.match(title)
    if not match:
        return None
    try:
        year, month, day = int(match.group(1)), int(match.group(2)), int(match.group(3))
    except ValueError:
        return None
    if not (2000 <= year <= 2099 and 1 <= month <= 12 and 1 <= day <= 31):
        return None
    try:
        return datetime(year, month, day, tzinfo=timezone.utc)
    except ValueError, OverflowError:
        return None


def normalize_sort_by(sort_by: str | None, *, allow_starred: bool = False) -> str:
    """Normalize a list sort key. ``post``/``received`` always; ``starred`` only
    when the caller opts in.

    *allow_starred* is not decoration. "starred" orders by when the item was
    starred (saved_entries.saved_at) and exists for Read Mode's Inbox — a to-do
    pile is newest-first by when you added to it. Blessing it globally let it
    reach the main index, which **persists** whatever it is handed as the
    remembered sort; the regular sort menu has no entry for "starred", so nothing
    rendered as active and the toolbar showed "Published newest" while the list
    was ordered by star date. Reported as the Feed view reverting to "Pub new"
    after switching in and out of e-ink mode.
    """
    # "size" (Saved/Kept item size, decided 2026-08-24) is gated the same way
    # as "starred" -- both mean something only in a star/kept-scoped view.
    if sort_by in {"post", "received"} or (allow_starred and sort_by in {"starred", "size"}):
        return sort_by
    return DEFAULT_SORT_BY


def normalize_sort_dir(sort_dir: str | None) -> str:
    if sort_dir in {"asc", "desc"}:
        return sort_dir
    return DEFAULT_SORT_DIR


def normalize_read_filter(read_filter: str | None) -> str:
    # Keep legacy values working while consolidating into all/unread/history modes.
    # "starred" is independent of read state entirely (see list_entries_for_feeds)
    # and deliberately not part of the get_next_read_filter cycle — reachable only
    # from the filter dropdown, not the cycle shortcut.
    if read_filter in {"all", "unread", "history", "starred", "capture_failed"}:
        return read_filter
    if read_filter == "read":
        return "history"
    if read_filter == "saved":
        return "all"
    return "all"


def normalize_resume_read_filter(read_filter: str | None) -> str:
    if read_filter in {"all", "unread"}:
        return read_filter
    return "all"


def get_next_read_filter(read_filter: str) -> str:
    return {
        "all": "unread",
        "unread": "history",
        "history": "all",
        "capture_failed": "all",
    }[normalize_read_filter(read_filter)]


def normalize_star_only(star_only: str | int | bool | None) -> bool:
    if isinstance(star_only, bool):
        return star_only
    if isinstance(star_only, int):
        return star_only != 0
    if isinstance(star_only, str):
        return star_only.strip().lower() in {"1", "true", "yes", "on"}
    return False


def build_star_only_query(star_only: str | int | bool | None) -> str:
    return "&star_only=1" if normalize_star_only(star_only) else ""


def build_sort_query(sort_by: str | None, sort_dir: str | None) -> str:
    """Return URL fragment for sort params, omitting defaults (post/asc)."""
    sb = normalize_sort_by(sort_by)
    sd = normalize_sort_dir(sort_dir)
    parts = []
    if sb != DEFAULT_SORT_BY:
        parts.append(f"&sort_by={quote_plus(sb)}")
    if sd != DEFAULT_SORT_DIR:
        parts.append(f"&sort_dir={quote_plus(sd)}")
    return "".join(parts)


def build_read_filter_query(read_filter: str | None) -> str:
    """Return URL fragment for read_filter, omitting default 'all'."""
    rf = normalize_read_filter(read_filter)
    return f"&read_filter={quote_plus(rf)}" if rf != "all" else ""


def build_resume_read_filter_query(read_filter: str | None, *, active_read_filter: str | None = None) -> str:
    """Return URL fragment for resume_read_filter, omitting when same as active filter."""
    rrf = normalize_resume_read_filter(read_filter)
    active = normalize_read_filter(active_read_filter) if active_read_filter is not None else "all"
    return f"&resume_read_filter={quote_plus(rrf)}" if rrf != active else ""


def upsert_entry_read_state(feed_url: str, entry_id: str, read_at: datetime | None = None) -> None:
    when = (read_at or datetime.now()).isoformat()
    with get_meta_connection() as conn:
        conn.execute(
            """
            INSERT INTO entry_read_state (feed_url, entry_id, read_at)
            VALUES (?, ?, ?)
            ON CONFLICT(feed_url, entry_id) DO UPDATE SET read_at = excluded.read_at
            """,
            (feed_url, entry_id, when),
        )


def delete_entry_read_state(feed_url: str, entry_id: str) -> None:
    with get_meta_connection() as conn:
        conn.execute(
            "DELETE FROM entry_read_state WHERE feed_url = ? AND entry_id = ?",
            (feed_url, entry_id),
        )


READ_HISTORY_CAP = 2_000


def append_read_history(
    feed_url: str,
    entry_id: str,
    title: str,
    link: str,
    feed_title: str,
) -> None:
    now = datetime.now().isoformat()
    with get_meta_connection() as conn:
        conn.execute(
            "INSERT INTO read_history (feed_url, entry_id, title, link, feed_title, read_at)"
            " VALUES (?, ?, ?, ?, ?, ?)"
            " ON CONFLICT(feed_url, entry_id) DO UPDATE SET"
            "  title=excluded.title, link=excluded.link,"
            "  feed_title=excluded.feed_title, read_at=excluded.read_at",
            (feed_url, entry_id, title, link, feed_title, now),
        )
        conn.execute(
            "DELETE FROM read_history WHERE id NOT IN (SELECT id FROM read_history ORDER BY read_at DESC LIMIT ?)",
            (READ_HISTORY_CAP,),
        )


_IMG_ATTR_RE = re.compile(r'([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*=\s*["\']([^"\']+)["\']')
_DIV_TAG_RE = re.compile(r"<(/?)div\b[^>]*>", re.IGNORECASE)
_AUDIO_SRC_RE = re.compile(r'<audio\b[^>]*\bsrc=["\']([^"\']+)["\']', re.IGNORECASE)
_KG_AUDIO_CARD_RE = re.compile(r"<div\b[^>]*\bkg-audio-card\b[^>]*>", re.IGNORECASE)

# Webcomic navigation button detection — alt text and src filename patterns.
_COMIC_NAV_ALT_RE = re.compile(
    r'\balt=["\'](?:prev(?:ious)?|next(?:\s+page)?|first|last|back|forward|newer|older|beginning)["\']',
    re.IGNORECASE,
)
_COMIC_NAV_SRC_RE = re.compile(
    r'src=["\'][^"\']*[/_-](?:prev(?:ious)?|next|first|last|nav|back)[^"\']*\.(?:png|gif|jpe?g|svg)["\']',
    re.IGNORECASE,
)


def _strip_comic_nav_images(content: str) -> str:
    """Strip <a href="..."><img alt="prev/next/..."></a> navigation button combos from webcomic feeds."""

    def _check_nav(m: re.Match) -> str:
        tag = m.group(0)
        if _COMIC_NAV_ALT_RE.search(tag) or _COMIC_NAV_SRC_RE.search(tag):
            return ""
        return tag

    result = re.sub(
        r"<a\b[^>]*>(?:\s*<img\b[^>]*/?\s*>\s*)+</a>",
        _check_nav,
        content,
        flags=re.IGNORECASE | re.DOTALL,
    )
    # Clean up empty <p> blocks left behind
    result = re.sub(r"<p\b[^>]*>\s*</p>", "", result, flags=re.IGNORECASE)
    return result


def _strip_qwantz_nav(content: str) -> str:
    """Strip site-nav table and prev/next row from Dinosaur Comics (qwantz.com) entries."""
    # Remove the site-nav table that appears before the comic image
    content = re.sub(
        r"<table\b[^>]*>.*?</table>\s*(?=<img\b)",
        "",
        content,
        count=1,
        flags=re.IGNORECASE | re.DOTALL,
    )
    # Remove the table row containing the prev/next navigation links (rel="prev" anchor)
    content = re.sub(
        r'<tr\b[^>]*>.*?rel=["\']prev["\'].*?</tr>',
        "",
        content,
        flags=re.IGNORECASE | re.DOTALL,
    )
    return content


def _strip_div_blocks_by_class(html: str, *class_markers: str) -> str:
    """Remove every <div> block whose opening tag's class contains all of class_markers.

    Uses depth-tracking to find the matching closing </div> so nested divs are
    handled correctly.  Multiple non-overlapping matches are all stripped.
    """
    open_re = re.compile(
        r'<div\b[^>]+class=["\'][^"\']*' + r'[^"\']*'.join(re.escape(m) for m in class_markers) + r'[^"\']*["\'][^>]*>',
        re.IGNORECASE,
    )
    result: list[str] = []
    pos = 0
    for match in open_re.finditer(html):
        start = match.start()
        if start < pos:
            continue
        result.append(html[pos:start])
        depth = 0
        end = start
        for dm in _DIV_TAG_RE.finditer(html, start):
            if dm.group(1):  # closing tag
                depth -= 1
                if depth == 0:
                    end = dm.end()
                    break
            else:
                depth += 1
        pos = end if end > start else match.end()
    result.append(html[pos:])
    return "".join(result)


# Chrome that only exists once the source page's own JavaScript runs. We don't
# run it, so a share widget stays a row of dead icons and a lazy "related posts"
# carousel stays a row of empty bullets with a spinner in each — observed on
# paizo.com, whose post ends with div.sharing_widget (anchors with no href at
# all) followed by four <li class="blog-item loading"> holding only a dice
# glyph. Same class of problem as the JWPlayer chrome strip below.
_JS_PLACEHOLDER_CLASS_RE = re.compile(r"\b(?:loading|placeholder|skeleton)\b", re.IGNORECASE)
_SHARE_WIDGET_CLASS_RE = re.compile(r"\b(?:shar(?:e|ing)[-_ ]?\w*|social(?:[-_ ]?\w*)?)\b", re.IGNORECASE)
# Icon signals for inline SVG: Font Awesome's own marker plus the generic
# conventions. Mirrors the sizing rules in reader/style CSS so "what counts as an
# icon" has one answer. Anything at or above this many px in a declared
# width/height is art, not a glyph (same floor as the lead-image scanner).
_SVG_ICON_CLASS_RE = re.compile(r"\bsvg-inline--fa\b|\bicon\b|[-_]icon\b|\bicon[-_]", re.IGNORECASE)
_SVG_ICON_MAX_PX = 64


def _strip_js_dependent_chrome(html: str) -> str:
    """Drop share widgets and unfilled lazy-load placeholders from entry content.

    **Only ever removes elements that contain no text and no <img>.** That is the
    whole safety argument: a real "related posts" block carries headlines, and a
    real gallery carries images, so neither can match. What is left to match is
    icon-only chrome, which carries nothing a reader would read.
    """
    if not html or ("class" not in html):
        return html
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(html, "html.parser")
    changed = False

    def _svg_is_icon(svg) -> bool:
        """True for an inline SVG that is a UI glyph rather than article art.

        Inline SVG *is* a legitimate kind of post image, so it has to count as
        content — otherwise a chart or diagram in a container whose class merely
        matched would be silently stripped. But the chrome this function exists
        to remove is icon-only: paizo's share widget holds Font Awesome glyphs
        and its carousel placeholders hold a dice spinner. So icons don't count,
        and anything else does. Same signals the stylesheet uses to size icons.
        """
        classes = " ".join(svg.get("class") or [])
        if _SVG_ICON_CLASS_RE.search(classes):
            return True
        # Otherwise judge by declared size: a glyph pins small px dimensions,
        # art does not. Note Font Awesome's are viewBox units (320x512), which
        # is exactly why the class test above has to come first.
        sizes: list[int] = []
        for attr in ("width", "height"):
            raw = str(svg.get(attr) or "").strip()
            if raw.isdigit():
                sizes.append(int(raw))
        if not sizes:
            return False  # nothing to judge by — assume art, keep it
        return max(sizes) < _SVG_ICON_MAX_PX

    def _is_textless_chrome(el) -> bool:
        if el.get_text(strip=True):
            return False
        if el.find("img") is not None:
            return False
        # An inline SVG that isn't an icon is article art — keep the container.
        return not any(not _svg_is_icon(sv) for sv in el.find_all("svg"))

    for el in soup.find_all(["div", "ul", "nav", "section", "aside", "li", "p"]):
        # A chrome container can hold another (paizo: ul.loading > li.loading).
        # find_all snapshots the tree, so the inner one is still in this list
        # after its parent was decomposed — and reading attrs off a decomposed
        # tag raises.
        if getattr(el, "decomposed", False):
            continue
        classes = " ".join(el.get("class") or [])
        if not classes:
            continue
        is_share = _SHARE_WIDGET_CLASS_RE.search(classes) is not None
        is_placeholder = _JS_PLACEHOLDER_CLASS_RE.search(classes) is not None
        if (is_share or is_placeholder) and _is_textless_chrome(el):
            el.decompose()
            changed = True

    # A list whose every item was a placeholder is now an empty <ul> — drop the
    # husk too, or the reader shows a stray bullet-less gap.
    for lst in soup.find_all(["ul", "ol"]):
        if getattr(lst, "decomposed", False):
            continue
        if not lst.find(["li"]) and not lst.get_text(strip=True) and lst.find("img") is None:
            lst.decompose()
            changed = True

    return str(soup) if changed else html


# WordPress "rss_footer" boilerplate appended to feed content: "The post <title>
# appeared first on <site>." Plugins add near-duplicate variants ("first appeared
# on") and some double-encode the wrapping <p> (so the reader shows literal
# "<p>...</p>" text). Match the paragraph text tolerantly of leading/trailing
# literal p-tags, anchored on "The post … appeared … on …".
_WP_POST_FOOTER_RE = re.compile(
    r"^\s*(?:</?p>\s*)*The post\b.*?\b(?:first appeared|appeared first)\s+on\b.*?\.\s*(?:</?p>\s*)*$",
    re.IGNORECASE | re.DOTALL,
)


def _fix_wp_post_footer(content_html: str) -> str:
    """Tidy the trailing WordPress "The post … appeared … on …" footer.

    The footer itself is fine to keep, but feeds mangle it two ways: plugins emit
    it twice ("first appeared on" + "appeared first on"), and some double-encode
    the wrapping <p> so it renders as literal "<p>…</p>" text. Keep a single
    footer paragraph, drop the duplicates, and strip the literal tag artifacts
    from the one kept. Only trailing <p> blocks are touched (stop at the first
    non-footer paragraph), so real content is never affected."""
    if "appeared" not in content_html.lower():
        return content_html  # cheap guard
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(content_html, "html.parser")
    paras = soup.find_all("p")
    footer_ps = []
    for p in reversed(paras):
        if p.parent is None:
            continue
        if _WP_POST_FOOTER_RE.match(p.get_text(" ", strip=True)):
            footer_ps.append(p)
        else:
            break  # contiguous trailing footers only
    if not footer_ps:
        return content_html
    footer_ps.reverse()  # document order
    keep = footer_ps[0]
    for dup in footer_ps[1:]:
        dup.decompose()
    # Remove literal "<p>"/"</p>" text artifacts (double-encoded tags) from the
    # kept footer, leaving the sentence intact.
    from bs4 import NavigableString as _NS

    for s in list(keep.strings):
        if not isinstance(s, _NS):
            continue
        cleaned = re.sub(r"</?p\s*>", "", str(s), flags=re.IGNORECASE)
        if cleaned != str(s):
            s.replace_with(cleaned)
    return str(soup)


def _clean_qwantz_content(content_html: str) -> str:
    """Strip Dinosaur Comics (qwantz.com) nav chrome, keep comic + commentary.

    The feed wraps the comic in ``<center>`` with a nav table above the image
    (archive / contact / merch / search / about) and another below it whose first
    row is prev/date/next nav and whose second row is the dated author
    commentary. Rebuild the body as just the comic ``<img>`` (its ``title`` holds
    the secret hover text, kept for the caption) followed by that commentary, so
    the duplicated nav and date links don't clutter the reading view."""
    if "qwantz.com" not in content_html and 'class="comic"' not in content_html:
        return content_html
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(content_html, "html.parser")
    img = soup.find("img", class_="comic") or soup.find("img")
    if img is None:
        return content_html  # not the shape we expect; leave untouched
    # The commentary lives in the only wide (colspan=3) cell with real text.
    commentary_html = ""
    for td in soup.find_all("td"):
        if str(td.get("colspan")) == "3" and td.get_text(strip=True):
            commentary_html = "".join(str(c) for c in td.contents)
            break
    return str(img) + commentary_html


_AUDIO_EXTS = (".mp3", ".m4a", ".m4b", ".aac", ".ogg", ".oga", ".opus", ".wav", ".flac")


def _url_has_audio_ext(url: str) -> bool:
    """True if the URL's path ends in a known audio extension.

    Matches on the path only, so a tracking/auth query string (e.g.
    ``…/ep1.mp3?token=abc``) — common on podcast CDNs — doesn't defeat the
    extension check the way a naive endswith on the whole URL would.
    """
    if not url:
        return False
    # Trim surrounding whitespace some feeds leave around URLs, which would
    # otherwise land in the parsed path and defeat the extension check.
    return urlparse(url.strip()).path.lower().endswith(_AUDIO_EXTS)


_IMAGE_EXTS = (".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", ".bmp", ".svg")


def _url_has_image_ext(url: str) -> bool:
    """True if the URL's path ends in a known image extension (query string
    tolerated, like _url_has_audio_ext)."""
    if not url:
        return False
    return urlparse(url.strip()).path.lower().endswith(_IMAGE_EXTS)


_BUZZSPROUT_ENCLOSURE_RE = re.compile(r"^(https://(?:www\.)?buzzsprout\.com/\d+/episodes/[^?#]+)\.mp3", re.IGNORECASE)


def _derived_entry_link(entry) -> str | None:
    """Best-effort page URL for entries whose feed ships no ``<link>``.

    Buzzsprout podcast feeds carry only a guid + audio enclosure; the episode page
    is the enclosure URL without the ``.mp3`` extension, so the post title can link
    somewhere instead of being inert."""
    if getattr(entry, "link", None):
        return None
    for enc in getattr(entry, "enclosures", None) or []:
        url = (getattr(enc, "href", None) or getattr(enc, "url", None) or "").strip()
        m = _BUZZSPROUT_ENCLOSURE_RE.match(url)
        if m:
            return m.group(1)
    return None


def _find_entry_audio_url(entry) -> str | None:
    """Return a usable audio URL for an entry, or None.

    Checks, in order: audio enclosures (by MIME type, or an audio extension on
    the URL path so query strings and untyped/oddly-typed enclosures still
    match), then the entry link when it points straight at an audio file (some
    podcast feeds carry no enclosure and set the link to the media URL).

    Note: audio that lives only in ``<media:content>`` is not recoverable here —
    the reader library keeps standard ``<enclosure>`` elements but drops
    media:content, so it never reaches this entry object.
    """
    for enc in getattr(entry, "enclosures", None) or []:
        enc_url = getattr(enc, "href", None) or getattr(enc, "url", None) or ""
        enc_type = (getattr(enc, "type", None) or "").lower()
        if enc_url and (enc_type.startswith("audio/") or _url_has_audio_ext(enc_url)):
            return enc_url
    link = getattr(entry, "link", None) or ""
    if _url_has_audio_ext(link):
        return link
    return None


def _format_enclosure_size(length) -> str:
    """Render an enclosure ``length`` (bytes) as a short human label, or ''.

    Feeds often set a placeholder length (e.g. ``1024``) when the real size is
    unknown, so anything under ~2 KB is treated as meaningless and dropped.
    """
    try:
        size = int(length)
    except TypeError, ValueError:
        return ""
    if size < 2048:
        return ""
    for unit in ("KB", "MB", "GB"):
        size /= 1024.0
        if size < 1024 or unit == "GB":
            return f"{size:.1f} {unit}"
    return ""


def _enclosure_label(enc_url: str, enc_type: str) -> str:
    """Friendly label for a download link: the file name, else the MIME type."""
    name = unquote(urlparse(enc_url).path.rsplit("/", 1)[-1]).strip()
    if name:
        return name
    return enc_type or "attachment"


def attachment_filename_for_url(source_url: str) -> str:
    """The filename a saved asset should download as.

    An archived asset is addressed by content hash, so without this the browser
    names the download after the URL's last segment — a 64-character hash with
    no extension, which is unopenable and unidentifiable. Taken from the source
    URL's own basename, minus any query.
    """
    name = urlparse(source_url or "").path.rsplit("/", 1)[-1]
    name = unquote(name).strip()
    # Header- and filesystem-hostile characters out; keep it recognisable.
    name = re.sub(r'[\\/:*?"<>|\r\n]+', "_", name).strip(". ")
    return name[:120] or "attachment"


def _attachment_list_item(source_url: str, label: str, meta: str, asset_map: dict[str, str]) -> str:
    """One <li>, pointing at the local copy when there is one.

    A saved post whose files still 404 at the publisher has kept the wrong half,
    so the archived copy wins whenever it exists; the badge says which you are
    getting, because "saved" is the reason the link still works.
    """
    asset_hash = asset_map.get(source_url)
    if asset_hash:
        href = html.escape(f"{STARRED_ASSET_URL_PREFIX}{asset_hash}", quote=True)
        badge = ' <span class="entry-attachment-saved" style="color:var(--muted,#888);font-size:0.85em;">saved</span>'
    else:
        href = html.escape(source_url, quote=True)
        badge = ""
    # `download` must carry the NAME: the archived URL ends in a content hash,
    # and a bare `download` makes the browser save that hash with no extension.
    dl = html.escape(attachment_filename_for_url(source_url), quote=True)
    # data-source-url (the ORIGINAL url, not the local-copy href above) and
    # data-kept let the Attachments panel's JS (loadEntryAttachments in
    # app.js) add a delete control to this exact row instead of re-rendering
    # it — the href alone isn't enough to identify the row once it points at
    # a content-hashed local copy.
    src_attr = html.escape(source_url, quote=True)
    kept_attr = ' data-kept="1"' if asset_hash else ""
    return (
        f'<li data-source-url="{src_attr}"{kept_attr}>'
        f'<a href="{href}" target="_blank" rel="noopener noreferrer" '
        f'download="{dl}">{label}</a>{meta}{badge}</li>'
    )


# Enclosure content types that describe a PAGE rather than a file. Deliberately
# short: text/plain and application/xml can each be a genuine attachment, and a
# type this list gets wrong disappears a real download.
_PAGE_ENCLOSURE_TYPES = frozenset(
    {
        "text/html",
        "application/xhtml+xml",
    }
)


def _url_ext(url: str) -> str:
    """Lowercased extension of a URL's PATH, or "" — query strings never count."""
    try:
        path = urlparse(url).path.lower()
    except ValueError:
        return ""
    dot = path.rfind(".")
    return path[dot + 1 :] if dot >= 0 else ""


def _filtered_file_enclosures(entry, audio_url: str | None) -> list:
    """Enclosures on *entry* that look like a real attached file, in order,
    deduped by URL -- shared by `_render_entry_attachments` (which lists
    every one, kept or not) and the Attachments panel's "available to save"
    scan (`entry_attachments_route`), which needs the exact same "is this
    actually a file" rule for enclosures that never appear as a body link at
    all (Full Circle Magazine's issue PDF/EPUB, a devblog's demo .mp4) --
    without this, un-keeping one left it with no way back into the panel:
    reported live 2026-09-13, twice, right after shipping the panel.

    Excludes audio (surfaced as the player), image (surfaced as the lead
    image), and page-typed enclosures (Standard Ebooks' "read online"
    single-page edition), and the audio enclosure already playing.
    """
    seen: set[str] = set()
    out: list = []
    for enc in getattr(entry, "enclosures", None) or []:
        enc_url = (getattr(enc, "href", None) or getattr(enc, "url", None) or "").strip()
        if not enc_url or enc_url in seen:
            continue
        enc_type = (getattr(enc, "type", None) or "").lower()
        if enc_type.startswith("audio/") or _url_has_audio_ext(enc_url):
            continue
        if enc_type.startswith("image/") or _url_has_image_ext(enc_url):
            continue
        if enc_type in _PAGE_ENCLOSURE_TYPES:
            continue
        if not enc_type and _url_ext(enc_url) in _NEVER_ATTACHMENT_EXTS:
            continue
        if audio_url and enc_url == audio_url:
            continue
        seen.add(enc_url)
        out.append(enc)
    return out


def _render_entry_attachments(entry, audio_url: str | None, asset_map: dict[str, str] | None = None) -> str:
    """Render a footer "Attachments" section for non-audio enclosures.

    Magazine/document feeds (e.g. Full Circle) attach the issue PDF/EPUB as
    ``<enclosure>`` elements that never appear in the article body. The audio
    enclosure (if any) is already surfaced as a player, so it's skipped here to
    avoid a duplicate link.

    *asset_map* (source_url -> hash) is the entry's archived assets. Anything in
    it is served from the local copy instead of the publisher's URL, so a saved
    post's files still open after the site goes down — the whole point of
    keeping them. Files captured from BODY LINKS (the per-feed extension list)
    are appended to the same list rather than getting a section of their own:
    from the reader's side they are the same thing, "files that came with this
    post", and the difference is only in how Lectio found them.
    """
    if asset_map is None:
        # Fetched here rather than threaded through: this renderer is called
        # from a path that has no map in hand, and the lookup is one indexed
        # query on the entry's own key.
        try:
            asset_map = starred_archive_service.get_entry_asset_map(str(entry.feed_url), str(entry.id))
        except Exception:  # noqa: BLE001 — falls back to publisher URLs
            asset_map = {}
    asset_map = asset_map or {}
    seen: set[str] = set()
    items: list[str] = []
    for enc in _filtered_file_enclosures(entry, audio_url):
        enc_url = (getattr(enc, "href", None) or getattr(enc, "url", None) or "").strip()
        if enc_url in seen:
            continue
        enc_type = (getattr(enc, "type", None) or "").lower()
        seen.add(enc_url)
        label = html.escape(_enclosure_label(enc_url, enc_type))
        size = _format_enclosure_size(getattr(enc, "length", None))
        meta = f' <span style="color:var(--muted,#888);">({size})</span>' if size else ""
        items.append(_attachment_list_item(enc_url, label, meta, asset_map))

    # Files captured from body links, appended to the same list.
    # Judged by STORED content type, not by guessing from the URL: a Gravatar
    # ("/avatar/<hash>?s=48") and a CDN path with no extension are both images
    # with nothing in the URL to say so, and they surfaced as "attachments".
    try:
        file_assets = starred_archive_service.get_entry_file_assets(str(entry.feed_url), str(entry.id))
    except Exception:  # noqa: BLE001
        file_assets = {}
    for source_url in file_assets:
        if source_url in seen or not source_url:
            continue
        seen.add(source_url)
        items.append(_attachment_list_item(source_url, html.escape(_enclosure_label(source_url, "")), "", asset_map))

    if not items:
        return ""
    # id/data-* here, not on a separate template element: this whole block is
    # baked into content_html at render time (see docstring), so it's the only
    # foothold the Attachments panel's JS (loadEntryAttachments in app.js) has
    # to find its feed_url/entry_id, add delete/save controls to existing rows,
    # and append "available but not kept" candidates the per-feed policy
    # doesn't cover -- see /entries/attachments and docs/architecture/saved.md.
    _feed_url_attr = html.escape(str(getattr(entry, "feed_url", "") or ""), quote=True)
    _entry_id_attr = html.escape(str(getattr(entry, "id", "") or ""), quote=True)
    return (
        f'<div id="entry-attachments" class="entry-attachments" '
        f'data-feed-url="{_feed_url_attr}" data-entry-id="{_entry_id_attr}" '
        'style="margin:1.5em 0 0.5em; padding-top:0.75em; border-top:1px solid var(--border,#ddd);">'
        '<div id="entry-attachments-header" style="display:flex;align-items:center;'
        'justify-content:space-between;gap:0.5em;margin-bottom:0.35em;">'
        '<span style="font-weight:600;">Attachments</span></div>'
        f'<ul id="entry-attachments-list" style="margin:0; padding-left:1.25em;">{"".join(items)}</ul>'
        "</div>"
    )


# media:content scan cadence: rescan podcast feeds (audio found) often enough to
# catch new episodes; barely ever rescan feeds that had none.
_MEDIA_SCAN_TTL_FOUND = 6 * 3600
_MEDIA_SCAN_TTL_EMPTY = 7 * 24 * 3600


def _lookup_media_audio(conn: sqlite3.Connection, feed_url: str, entry_id: str) -> str | None:
    row = conn.execute(
        "SELECT audio_url FROM entry_media_audio WHERE feed_url = ? AND entry_id = ?",
        (feed_url, entry_id),
    ).fetchone()
    return (row[0] or None) if row else None


def _lookup_media_video(conn: sqlite3.Connection, feed_url: str, entry_id: str) -> list[str] | None:
    """Recovered YouTube ids for an entry: ``None`` = not scanned yet, ``[]`` =
    scanned but nothing recoverable, otherwise the ids in document order."""
    row = conn.execute(
        "SELECT video_ids FROM entry_media_video WHERE feed_url = ? AND entry_id = ?",
        (feed_url, entry_id),
    ).fetchone()
    if row is None:
        return None
    return list(str(row[0] or "").split())


def _media_scan_due(conn: sqlite3.Connection, feed_url: str) -> bool:
    """True if this feed has never been scanned for media:content audio, or its
    cadence TTL has elapsed."""
    row = conn.execute("SELECT scanned_at, found, ok FROM feed_media_scan WHERE feed_url = ?", (feed_url,)).fetchone()
    if not row:
        return True
    if row[1]:
        ttl = _MEDIA_SCAN_TTL_FOUND
    elif not row[2]:  # errored mid-scan — not proof of "no audio"
        ttl = _MEDIA_SCAN_TTL_ERROR
    else:
        ttl = _MEDIA_SCAN_TTL_EMPTY
    return (time.time() - float(row[0])) > ttl


def _scan_feed_media_audio(feed_url: str) -> None:
    """Fetch the raw feed (SSRF-guarded) and persist any media:content audio URLs.

    Runs in a background thread; the caller captures and re-binds tenancy. Best
    effort — a fetch/parse failure still records the scan attempt so we back off."""
    found_map: dict[str, str] = {}
    raw_feed: bytes | None = None
    try:
        with url_guard.build_client(timeout=8.0, headers={"User-Agent": READABILITY_USER_AGENT}) as client:
            resp = url_guard.safe_get(client, feed_url)
        if resp.status_code == 200 and resp.content:
            raw_feed = resp.content
            found_map = podcast_audio.extract_media_audio(raw_feed)
    except Exception:
        LOGGER.debug("media:content scan failed for %s", feed_url, exc_info=True)

    # The same raw feed also carries any YouTube embeds that feedparser/reader
    # stripped from the stored content (the <iframe>, and thus the video id, is
    # gone). Recover them here so the pane can rebuild the player.
    video_map: dict[str, list[str]] = {}
    if raw_feed is not None:
        try:
            video_map = youtube_embeds.extract_youtube_embeds(raw_feed)
        except Exception:
            LOGGER.debug("youtube-embed scan failed for %s", feed_url, exc_info=True)

    # When the feed carries no audio of its own, the audio often lives in a
    # separate podcast-host feed (Libsyn/Buzzsprout/…) referenced on the episode
    # page. Detect it so we can (a) borrow its audio into this feed's entries and
    # (b) suggest subscribing to the audio feed if borrowing didn't cover an entry.
    suggested = ""
    errored = False
    if not found_map:
        suggested = _discover_suggested_audio_feed(feed_url)
        if suggested:
            borrowed = _borrow_audio_from_feed(feed_url, suggested)
            if borrowed is None:
                # We found a host feed but couldn't fetch it (network/HTTP error,
                # e.g. a Cloudflare 403) — don't bank this as a settled "no audio".
                errored = True
            else:
                found_map = borrowed

    # ``ok`` is False only when we have positive reason to believe the scan was
    # incomplete; a clean scan that simply found nothing is ok=1 (long backoff).
    ok = 0 if (errored and not found_map) else 1
    with get_meta_connection() as conn:
        for entry_id, audio_url in found_map.items():
            conn.execute(
                "INSERT INTO entry_media_audio (feed_url, entry_id, audio_url) VALUES (?, ?, ?)"
                " ON CONFLICT(feed_url, entry_id) DO UPDATE SET audio_url = excluded.audio_url",
                (feed_url, entry_id, audio_url),
            )
        for entry_id, vids in video_map.items():
            conn.execute(
                "INSERT INTO entry_media_video (feed_url, entry_id, video_ids) VALUES (?, ?, ?)"
                " ON CONFLICT(feed_url, entry_id) DO UPDATE SET video_ids = excluded.video_ids",
                (feed_url, entry_id, " ".join(vids)),
            )
        conn.execute(
            "INSERT INTO feed_media_scan (feed_url, scanned_at, found, suggested_audio_feed, ok)"
            " VALUES (?, ?, ?, ?, ?)"
            " ON CONFLICT(feed_url) DO UPDATE SET scanned_at = excluded.scanned_at,"
            " found = excluded.found, suggested_audio_feed = excluded.suggested_audio_feed,"
            " ok = excluded.ok",
            (feed_url, time.time(), 1 if found_map else 0, suggested, ok),
        )
        conn.commit()


def _polite_safe_get(url: str, *, timeout: float):
    """SSRF-safe GET that identifies honestly as Lectio, escalating to a browser
    UA only if the honest request is actually refused (HTTP 403).

    Honest-by-default keeps us a good citizen for hosts that reward it (and never
    spoofs a host that's happy to serve Lectio); the browser-UA fallback is a
    last resort for Cloudflare-gated podcast hosts that blanket-403 non-browsers.
    Returns the final response (caller checks status)."""
    resp = None
    for ua in (LECTIO_HONEST_USER_AGENT, PODCAST_FETCH_USER_AGENT):
        with url_guard.build_client(timeout=timeout, headers={"User-Agent": ua}) as client:
            resp = url_guard.safe_get(client, url)
        if resp.status_code != 403:
            break  # served (or a non-403 error) — don't escalate
    return resp


def _discover_suggested_audio_feed(feed_url: str) -> str:
    """Fetch a recent entry's page and look for a podcast-host audio feed.

    Returns the discovered feed URL (empty string if none). Best effort; the feed
    itself isn't re-fetched — we use a recent entry's link page, where the host
    player embed / feed URL lives."""
    try:
        with get_reader() as reader:
            link = ""
            for entry in reader.get_entries(feed=feed_url, sort="recent"):
                link = (entry.link or "").strip()
                if link:
                    break
        if not link or not url_guard.is_safe_outbound_url(link):
            return ""
        resp = _polite_safe_get(link, timeout=8.0)
        if resp.status_code != 200 or not resp.text:
            return ""
        found = podcast_feed_discovery.find_podcast_host_feed(resp.text) or ""
        # Don't suggest the feed we're already looking at.
        if found and found.rstrip("/") == feed_url.rstrip("/"):
            return ""
        return found
    except Exception:
        LOGGER.debug("audio-feed discovery failed for %s", feed_url, exc_info=True)
        return ""


def _borrow_audio_from_feed(feed_url: str, host_feed_url: str) -> dict[str, str] | None:
    """Match this feed's entries to audio in a podcast-host feed and return
    entry_id -> audio URL for the matches.

    Lets a notes-only website feed gain a player without a second subscription:
    the audio is sourced from the matching episode (by title, then episode
    number) in the linked podcast feed.

    Returns ``None`` when the host feed couldn't be fetched (network/HTTP error)
    — distinct from an empty dict, which means it was fetched fine but nothing
    matched. The caller uses that to retry a failed fetch soon rather than
    backing it off as a settled "no audio" result."""
    try:
        with get_reader() as reader:
            titles = {str(e.id): (e.title or "") for e in reader.get_entries(feed=feed_url, sort="recent", limit=120)}
        if not titles or not url_guard.is_safe_outbound_url(host_feed_url):
            return {}
        resp = _polite_safe_get(host_feed_url, timeout=10.0)
        if resp.status_code != 200 or not resp.content:
            return None
        return podcast_feed_discovery.match_episode_audio(resp.content, titles)
    except Exception:
        LOGGER.debug("audio borrow failed for %s from %s", feed_url, host_feed_url, exc_info=True)
        return None


def _get_suggested_audio_feed(conn: sqlite3.Connection, feed_url: str) -> str | None:
    row = conn.execute("SELECT suggested_audio_feed FROM feed_media_scan WHERE feed_url = ?", (feed_url,)).fetchone()
    return (row[0] or None) if row else None


def _is_feed_subscribed(conn: sqlite3.Connection, feed_url: str) -> bool:
    """True if the URL (slash-insensitive) is in any folder."""
    fu = feed_url.rstrip("/")
    row = conn.execute(
        "SELECT 1 FROM folder_feeds WHERE feed_url = ? OR feed_url = ? LIMIT 1",
        (fu, fu + "/"),
    ).fetchone()
    return bool(row)


def _queue_media_audio_scan(feed_url: str) -> None:
    """Background-scan a feed for media:content audio (deduped, tenancy-bound).

    Honours LECTIO_DISABLE_STARTUP_BACKFILL for the reason conftest sets it: a
    daemon writing the per-user meta DB races the test's own ops on the same
    temp DB and surfaces as an intermittent "database is locked". That switch
    already covered the *startup* daemons; this one is spawned per request, so
    it slipped through and kept failing CI from a test that never touches it
    (test_feed_link_xss, which only renders a post list).
    """
    if os.getenv("LECTIO_DISABLE_STARTUP_BACKFILL", "0") == "1":
        return
    uid = tenancy.current_user_id()
    key = (uid, feed_url)
    with _media_scan_lock:
        if key in _media_scan_in_progress:
            return
        _media_scan_in_progress.add(key)

    def _bg() -> None:
        try:
            with tenancy.user_context(uid):
                _scan_feed_media_audio(feed_url)
        finally:
            with _media_scan_lock:
                _media_scan_in_progress.discard(key)

    threading.Thread(target=_bg, daemon=True).start()


def _resolve_entry_audio_url(
    conn: sqlite3.Connection,
    feed_url: str,
    entry_id: str,
    entry,
) -> str | None:
    """Standard enclosure/link audio detection, with a media:content fallback.

    The fallback is cached per entry; on a cache miss for a feed that's due a
    scan, a background re-parse is enqueued (so the article open never blocks)
    and the player fills in on a later open."""
    direct = _find_entry_audio_url(entry)
    if direct:
        return direct
    cached = _lookup_media_audio(conn, feed_url, entry_id)
    if cached:
        return cached
    if _media_scan_due(conn, feed_url):
        _queue_media_audio_scan(feed_url)
    return None


def _transform_kg_audio_cards(content_html: str) -> str:
    """Replace Ghost CMS kg-audio-card widgets with plain <audio controls> elements.

    Ghost's custom player relies on JS to format the duration (stored as raw seconds)
    and to wire up play/pause controls. Without the JS it renders as a broken UI with
    values like '518.82585'. This replaces the whole card div with a native audio player.
    """
    result: list[str] = []
    pos = 0
    for card_m in _KG_AUDIO_CARD_RE.finditer(content_html):
        result.append(content_html[pos : card_m.start()])
        # Walk div open/close tags from here to find the matching end tag.
        depth = 1
        end_pos = len(content_html)
        for tag_m in _DIV_TAG_RE.finditer(content_html, card_m.end()):
            if tag_m.group(1):  # closing </div>
                depth -= 1
                if depth == 0:
                    end_pos = tag_m.end()
                    break
            else:
                depth += 1
        card_html = content_html[card_m.start() : end_pos]
        src_m = _AUDIO_SRC_RE.search(card_html)
        if src_m:
            src = html.escape(src_m.group(1), quote=True)
            result.append(f'<audio controls preload="metadata" src="{src}" style="width:100%"><a href="{src}">Download audio</a></audio>')
        else:
            result.append(card_html)
        pos = end_pos
    result.append(content_html[pos:])
    return "".join(result)


# HTML sanitization lives in services.html_sanitize — the single allowlist
# chokepoint shared by feed ingest (services.reader_sanitize) and the
# readability/source-HTML paths below. It keeps safe embeds (iframes from a
# curated host allowlist, sanitized SVG/MathML) instead of feedparser's
# destroy-everything behavior.
def _sanitize_html_allowlist(content: str) -> str:
    return html_sanitize.sanitize_html(content)


def sanitize_readability_html(content: str) -> str:
    """Sanitize Readability-extracted article HTML (rendered with `| safe`)."""
    return html_sanitize.sanitize_html(content)


def sanitize_source_html(content: str) -> str:
    """Sanitize proxied source-page HTML (rendered with `| safe`)."""
    return html_sanitize.sanitize_html(content)


def _reinject_readability_embeds(summary_html: str, raw_html: str) -> str:
    """Recover allowlisted <iframe> embeds that Readability stripped.

    `python-readability`'s `.summary()` drops *every* `<iframe>` during
    extraction, so YouTube/Spotify/Bandcamp players vanish from Reader view.
    Pull the allowlisted embeds out of the raw page and append any the extracted
    article is missing. The combined HTML is sanitized by the caller, so the
    re-injected iframes still pass through `_sanitize_iframe` (sandbox/referrer).
    """
    if "<iframe" not in raw_html.lower():
        return summary_html
    try:
        from bs4 import BeautifulSoup

        raw_soup = BeautifulSoup(raw_html, "html.parser")
    except Exception:
        return summary_html

    existing = summary_html.lower()
    seen: set[str] = set()
    missing: list[str] = []
    for ifr in raw_soup.find_all("iframe"):
        src = next((v for v in (str(ifr.get(a) or "").strip() for a in ("src", "data-src", "data-lazy-src")) if v), "")
        if not src or not html_sanitize._embed_host_allowed(src):
            continue
        key = src.split("?", 1)[0].lower()
        if key in seen or key in existing:
            continue
        seen.add(key)
        missing.append(str(ifr))
    if not missing:
        return summary_html
    block = "".join(f'<p class="lectio-embed">{m}</p>' for m in missing)
    return summary_html + block


def _replace_youtube_facades(raw_html: str) -> str:
    """Swap a publisher's YouTube click-to-play facade for a bare watch link, before extraction.

    guitarworld's facade is a thumbnail, a title, a play glyph and a "Watch On YouTube" logo link. Extraction kept all of it, so a
    re-fetch turned the playable video into a dead thumbnail. A bare link to the video is what the pane's embed placement already
    swaps for a real player (_place_recovered_embeds pass 1), and it stays a working link when no player can be recovered.
    """
    if "youtube-facade" not in raw_html:
        return raw_html
    try:
        from bs4 import BeautifulSoup

        soup = BeautifulSoup(raw_html, "html.parser")
        changed = False
        for facade in soup.find_all(class_="youtube-facade"):
            if facade.parent is None:
                continue
            ids = youtube_embeds._video_ids(str(facade))
            if not ids:
                continue
            url = f"https://www.youtube.com/watch?v={ids[0]}"
            link = soup.new_tag("a", href=url)
            link.string = url
            para = soup.new_tag("p")
            para.append(link)
            facade.replace_with(para)
            changed = True
        return str(soup) if changed else raw_html
    except Exception:  # noqa: BLE001 — never fail an extraction over cosmetics
        LOGGER.debug("youtube facade swap failed", exc_info=True)
        return raw_html


def _strip_site_chrome(raw_html: str, source_url: str) -> str:
    """Remove chrome a per-site plugin named, before extraction.

    ``_strip_article_chrome`` above is the generic list and runs on the
    readability path only — full-page capture keeps every node by design. That
    design has a cost a site can be specific about: basslessons ships its cookie
    banner as ``display: none`` (JS reveals it), first in the DOM, so a full-page
    capture opened with ~700 characters of cookie policy.
    """
    selectors = site_content_plugins.strip_selectors(source_url)
    if not selectors:
        return raw_html
    try:
        from bs4 import BeautifulSoup

        soup = BeautifulSoup(raw_html, "html.parser")
        removed = False
        for sel in selectors:
            for el in soup.select(sel):
                el.decompose()
                removed = True
        return str(soup) if removed else raw_html
    except Exception:  # noqa: BLE001 — never fail an extraction over cosmetics
        LOGGER.debug("site chrome strip failed for %s", source_url, exc_info=True)
        return raw_html


def _append_site_embeds(article_html: str, source_url: str, raw_html: str, *, prepend: bool = False) -> str:
    """Append a per-site embed the page only produces via JS.

    ``_reinject_readability_embeds`` above recovers embeds that were in the
    fetched HTML. This is the case it cannot reach: the element does not exist
    until the page calls its own resolver, so a plugin has to go and get it.

    Sanitized here rather than by the caller — this runs *after* extraction, so
    the block would otherwise be appended to already-sanitized output and never
    meet ``_sanitize_iframe``.
    """
    try:
        embed = site_content_plugins.extra_embed_html(source_url, raw_html)
    except Exception:  # noqa: BLE001 — an embed is a bonus, never a failure
        LOGGER.debug("site embed lookup failed for %s", source_url, exc_info=True)
        return article_html
    if not embed:
        return article_html
    clean = sanitize_readability_html(f'<p class="lectio-embed">{embed}</p>').strip()
    if not clean or "<iframe" not in clean.lower():
        return article_html  # the sanitizer rejected it — say nothing
    if not site_content_plugins.embed_at_top(source_url):
        return f"{article_html}{clean}"
    if prepend:
        # A caller-selected region has no title of its own; its first heading is a section label ("Tags").
        return f"{clean}{article_html}"
    # At the top means after the article's own heading, if it has one — above it
    # the video reads as a banner rather than as part of the piece.
    return _insert_after_first_heading(article_html, clean)


_FIRST_HEADING_RE = re.compile(r"</h[1-3]\s*>", re.IGNORECASE)


def _insert_after_first_heading(article_html: str, block: str) -> str:
    match = _FIRST_HEADING_RE.search(article_html)
    if match is None:
        return f"{block}{article_html}"
    return f"{article_html[: match.end()]}{block}{article_html[match.end() :]}"


_READABILITY_IMG_TAG_RE = re.compile(r"<img\b[^>]*>", re.IGNORECASE)
_READABILITY_IMG_SRC_RE = re.compile(r'\bsrc\s*=\s*["\']([^"\']+)["\']', re.IGNORECASE)


_DEDUPE_PRESENTATION_QUERY_KEYS = frozenset(
    {
        "width",
        "height",
        "w",
        "h",
        "quality",
        "q",
        "fit",
        "crop",
        "coordinates",
        "size",
        "dpr",
    }
)


def _dedupe_img_src_key(src: str) -> str:
    """Normalize an <img> src to a dedup key: path plus any query params NOT
    known to be pure presentation (resize/crop/quality) hints.

    Dropping the whole query string conflated two genuinely different images
    on any CDN that identifies the asset BY query param rather than by path
    — raised 2026-08-31: premierguitar.com's media-library CDN serves every
    image at the literal path .../image.jpg, distinguished only by ?id=, so
    two different tab-diagram images collided as "the same src" and the
    second was silently dropped as a duplicate. Presentation-only keys
    (width/height/quality/...) are still dropped, which is what lets the
    intended case — og:image and an in-body copy differing only by a resize
    query — keep deduping."""
    path, _, query = src.partition("?")
    if not query:
        return path.strip().lower()
    kept = sorted((k, v) for k, v in parse_qsl(query, keep_blank_values=True) if k.lower() not in _DEDUPE_PRESENTATION_QUERY_KEYS)
    if not kept:
        return path.strip().lower()
    return f"{path.strip().lower()}?{urlencode(kept)}"


def _dedupe_readability_images(article_html: str) -> str:
    """Drop repeated <img> tags that share an src (readability sometimes keeps
    the lead image twice — e.g. og:image plus the in-body copy)."""
    seen: set[str] = set()

    def _strip(match: re.Match[str]) -> str:
        src_match = _READABILITY_IMG_SRC_RE.search(match.group(0))
        if not src_match:
            return match.group(0)
        src = _dedupe_img_src_key(src_match.group(1))
        if src in seen:
            return ""
        seen.add(src)
        return match.group(0)

    return _READABILITY_IMG_TAG_RE.sub(_strip, article_html)


def _absolutize_article_urls(article_html: str, base_url: str) -> str:
    """Resolve relative ``src``/``href`` URLs in reader-view HTML against ``base_url``.

    The reader view is served from Lectio's own origin, so a relative image like
    ``model_m.webp`` (e.g. fabiensanglard.net, which uses page-relative image
    paths and ships no ``<base>`` tag) would otherwise resolve against Lectio and
    404. Readability's summary path absolutizes only when ``Document(url=...)`` is
    set; the BS4 content fallback returns its element verbatim, so apply one
    uniform pass here. Idempotent for already-absolute URLs."""
    if not article_html or not base_url:
        return article_html
    try:
        from lxml import html as lxml_html

        frag = lxml_html.fromstring(article_html)
        frag.make_links_absolute(base_url, resolve_base_href=False)
        return lxml_html.tostring(frag, encoding="unicode")
    except Exception:
        return article_html


def _set_or_replace_tag_attr(tag_html: str, attr_name: str, value: str) -> str:
    r"""Set *attr_name* on a tag, replacing it only if it is genuinely present.

    The boundary is ``(?<![-\w])``, not ``\b``: a hyphen is a non-word character,
    so ``\bsrc=`` matches inside ``data-lazy-src=``. Promoting a lazy image
    therefore "found" an existing src, rewrote the lazy attribute with its own
    value, and added no real src at all — the image had nothing to load and
    rendered as an empty box.
    """
    attr_re = re.compile(rf'(?<![-\w]){re.escape(attr_name)}\s*=\s*["\'][^"\']*["\']', re.IGNORECASE)
    attr_literal = f'{attr_name}="{html.escape(value, quote=True)}"'
    if attr_re.search(tag_html):
        return attr_re.sub(attr_literal, tag_html, count=1)
    insert_at = tag_html.rfind("/>")
    if insert_at != -1:
        return f"{tag_html[:insert_at]} {attr_literal}{tag_html[insert_at:]}"
    insert_at = tag_html.rfind(">")
    if insert_at != -1:
        return f"{tag_html[:insert_at]} {attr_literal}{tag_html[insert_at:]}"
    return f"{tag_html} {attr_literal}"


def normalize_proxy_lazy_media(content: str) -> str:
    """Promote common lazy-loading attrs so media renders without site scripts."""

    def _normalize_tag(match: re.Match[str]) -> str:
        tag = match.group(0)
        attrs: dict[str, str] = {}
        for attr_match in _IMG_ATTR_RE.finditer(tag):
            key = attr_match.group(1).strip().lower()
            value = html.unescape(attr_match.group(2).strip())
            if key and value:
                attrs[key] = value

        lower_tag = tag.lower()
        is_img = lower_tag.startswith("<img")
        is_source = lower_tag.startswith("<source")
        if not (is_img or is_source):
            return tag

        # data-runner-src: RebelMouse CMS (premierguitar.com and its sibling
        # sites) — raised 2026-08-31 against live "Ex. N" guitar-tab diagrams
        # shipping as a blank data:image/svg+xml placeholder src with the real
        # URL only in data-runner-src, so captures kept an empty box instead
        # of the tablature the lesson is actually about.
        lazy_src = (
            attrs.get("data-src")
            or attrs.get("data-lazy-src")
            or attrs.get("data-original")
            or attrs.get("data-image")
            or attrs.get("data-runner-src")
        )
        lazy_srcset = attrs.get("data-srcset") or attrs.get("data-lazy-srcset")
        current_src = attrs.get("src", "")
        current_srcset = attrs.get("srcset", "")

        # Common placeholders are 1x1 data URIs or empty src; replace with lazy attrs.
        placeholder_src = (not current_src) or current_src.startswith("data:")

        if is_img and lazy_src and placeholder_src:
            tag = _set_or_replace_tag_attr(tag, "src", lazy_src)

        if lazy_srcset and not current_srcset:
            tag = _set_or_replace_tag_attr(tag, "srcset", lazy_srcset)

        if is_source and lazy_src and (not current_src):
            tag = _set_or_replace_tag_attr(tag, "src", lazy_src)

        return tag

    return re.sub(r"<(?:img|source)\b[^>]*>", _normalize_tag, content, flags=re.IGNORECASE)


def _entry_query_suffix(feed_url: str, entry_id: str | None, include: bool = True) -> str:
    """Build the ``&feed_url=…&entry_id=…`` suffix used to re-select an entry after a
    redirect, or ``""`` when no entry should be selected. Centralizes the
    quote_plus pair that was repeated across the mark-read / range routes."""
    if not include or not entry_id:
        return ""
    return f"&feed_url={quote_plus(feed_url)}&entry_id={quote_plus(entry_id)}"


def add_no_referrer_to_images(content: str) -> str:
    """Add referrerpolicy="no-referrer" to inline <img> tags that lack it.

    Some sites (e.g. nanolx.org) serve a hotlink-protection placeholder image
    when the Referer header points at a foreign origin. The reader loads body
    images directly, so the browser sends the Lectio page URL as Referer and the
    real image is swapped for the placeholder. Suppressing the referer makes the
    request look like a direct hit, so hotlink-protected images load — and it
    avoids leaking the reader URL to image hosts.
    """
    if "<img" not in content.lower():
        return content
    return re.sub(
        r"<img\b(?![^>]*\breferrerpolicy\s*=)([^>]*?)(/?)>",
        r'<img\1 referrerpolicy="no-referrer"\2>',
        content,
        flags=re.IGNORECASE,
    )


# Mirrors the entry-lead-image handler in templates/_entry_pane.html. Kept as a
# literal rather than shared with the template because the template's copy is
# written inline in HTML and this one has to survive an attribute-level regex.
_IMG_PROXY_FALLBACK_ONERROR = (
    "var _s=this.getAttribute('src');"
    "if(!_s||_s.startsWith('/api/img?')||_s.startsWith('data:')){this.style.display='none'}"
    "else{this.setAttribute('src','/api/img?u='+encodeURIComponent(this.src))}"
)


def add_img_proxy_fallback(content: str) -> str:
    """Give body images the retry-through-/api/img the lead image already has.

    The article's hero has carried this since the proxy was added: if the direct
    load fails, swap src for the same-origin proxy, and only give up if that
    fails too. Body images had nothing — a failed load left a blank space with no
    second attempt and no way to tell a blocked image from one the publisher
    never shipped.

    That asymmetry is invisible until an article's og:image IS its body image.
    Then _strip_lead_image_opener correctly drops the separate hero (showing the
    picture twice is worse) and the only copy left is the body one — the copy
    with no fallback. sonarsource.com's blog does this on the posts that carry a
    screenshot, which is why the neighbours "showed their image" and one post
    did not: the neighbours were rendering a hero, not a body image.

    Cheap on the happy path: onerror never fires for an image that loads, so this
    costs nothing until something goes wrong. Only tags without an onerror are
    touched, and the sanitizer strips author-supplied handlers, so in practice
    that is all of them.
    """
    if "<img" not in content.lower():
        return content
    return re.sub(
        r"<img\b(?![^>]*\bonerror\s*=)([^>]*?)(/?)>",
        lambda m: f'<img{m.group(1)} onerror="{html.escape(_IMG_PROXY_FALLBACK_ONERROR, quote=True)}"{m.group(2)}>',
        content,
        flags=re.IGNORECASE,
    )


def proxy_hotlink_images(content: str) -> str:
    """Rewrite <img> src/srcset for hotlink-protected hosts to the /api/img proxy.

    referrerpolicy="no-referrer" fixes fresh loads, but a browser that already
    cached the host's "image was hotlinked" placeholder (under the unchanged
    image URL, with no Vary header) keeps serving it. Pointing src at
    /api/img?u=<url> is a new same-origin URL the browser hasn't cached, and the
    server-side fetch carries no Referer — so the real image loads. srcset is
    dropped for these images so the proxied src is the one used.
    """
    if "<img" not in content.lower():
        return content

    def _rewrite(m: re.Match) -> str:
        tag = m.group(0)
        src_m = re.search(r'\bsrc\s*=\s*(?:"([^"]*)"|\x27([^\x27]*)\x27)', tag, re.IGNORECASE)
        if not src_m:
            return tag
        src = html.unescape((src_m.group(1) or src_m.group(2) or "").strip())
        if not src or src.startswith("data:"):
            return tag
        if not _is_hotlink_img_host(urlparse(src).netloc):
            return tag
        proxied = f"/api/img?u={quote(src, safe='')}"
        tag = tag[: src_m.start()] + f'src="{proxied}"' + tag[src_m.end() :]
        # Drop srcset/data-* so the browser uses the proxied src, not a direct URL.
        tag = re.sub(
            r'\s+(?:srcset|data-srcset|data-src|data-lazy-src)\s*=\s*(?:"[^"]*"|\x27[^\x27]*\x27)',
            "",
            tag,
            flags=re.IGNORECASE,
        )
        return tag

    return re.sub(r"<img\b[^>]*>", _rewrite, content, flags=re.IGNORECASE)


def _run_rules_on_new_entries(feed_url: str) -> None:
    """Per-feed rules that act on unread entries, run as the feed's new entries land (late-bound: defined in services.automation_rules)."""
    for run in (_run_tag_filter_rules_for_feed, _run_mark_as_read_rules_for_feed):
        try:
            run(feed_url)
        except Exception:  # noqa: BLE001
            LOGGER.exception("[refresh] %s failed for %s", run.__name__, feed_url)


feed_refresh_service = FeedRefreshService(
    get_meta_connection=get_meta_connection,
    get_reader=get_reader,
    fetch_and_store_youtube_durations=youtube_duration_service.fetch_and_store_durations_for_feed,
    fetch_and_store_lead_images=lead_image_service.fetch_and_store_lead_images_for_feed,
    format_datetime_for_ui=format_datetime_for_ui,
    logger=LOGGER,
    refresh_debug_enabled=REFRESH_DEBUG_ENABLED,
    failed_feed_backoff_base_seconds=FAILED_FEED_BACKOFF_BASE_SECONDS,
    failed_feed_backoff_max_seconds=FAILED_FEED_BACKOFF_MAX_SECONDS,
    on_fetch_refused=_flag_browser_ua_on_refusal,
    on_fetch_still_blocked=_flag_proxy_feed_on_still_blocked,
    on_bot_challenge_still_blocked=_flag_flaresolverr_feed_on_still_blocked,
    on_fetch_still_blocked_via_proxy=_flag_tailscale_feed_on_still_blocked,
    on_proxy_unreachable=_mark_backend_unreachable,
    progress_hook=_note_scheduler_progress,
    on_new_entries=lambda feed_url: _run_rules_on_new_entries(feed_url),
)

websub_service: WebSubService | None = (
    WebSubService(
        get_shared_connection=get_websub_connection,
        public_url=LECTIO_PUBLIC_URL,
        user_agent=READABILITY_USER_AGENT,
        logger=LOGGER,
    )
    if LECTIO_PUBLIC_URL
    else None
)

# Auth is per-user via user_store; each request runs under a bound tenancy context.
fever_service: FeverService | None = FeverService(
    get_meta_connection=get_meta_connection,
    get_reader=get_reader,
    fever_api_key="",
    root_folder_name=ROOT_FOLDER_NAME,
    current_user=tenancy.current_user_id,
)

greader_service: GReaderService | None = GReaderService(
    get_meta_connection=get_meta_connection,
    get_reader=get_reader,
    username="",
    password=_FEVER_PASSWORD,
    root_folder_name=ROOT_FOLDER_NAME,
)

miniflux_service = MinifluxService(
    get_meta_connection=get_meta_connection,
    get_reader=get_reader,
    root_folder_name=ROOT_FOLDER_NAME,
    current_user=tenancy.current_user_id,
)


def build_source_proxy_response(source_url: str, picker: bool = False) -> HTMLResponse:
    parsed = urlparse(source_url)
    if parsed.scheme not in {"http", "https"}:
        return HTMLResponse("<h1>Unsupported URL scheme.</h1>", status_code=400)

    try:
        with url_guard.build_client(timeout=12.0, headers={"User-Agent": READABILITY_USER_AGENT}) as client:
            response = url_guard.safe_get(client, source_url, headers={"User-Agent": READABILITY_USER_AGENT})
        response.raise_for_status()
    except Exception as exc:
        escaped_url = html.escape(source_url)
        escaped_error = html.escape(public_error(exc, "Fetching the source"))
        return HTMLResponse(
            (
                "<!DOCTYPE html><html><head><meta charset='utf-8'><title>Source unavailable</title>"
                "<style>body{font-family:Segoe UI,Arial,sans-serif;margin:0;padding:1rem;line-height:1.45;}"
                "h1{font-size:1.05rem;margin:0 0 .5rem;}p{margin:.35rem 0;color:#555;}a{color:#0a5ca4;}</style></head>"
                f"<body><h1>Could not load the source page.</h1><p>{escaped_error}</p>"
                f"<p><a href='{escaped_url}' target='_blank' rel='noopener noreferrer'>Open original page</a></p></body></html>"
            ),
            status_code=200,
        )

    raw_html = response.text

    # Detect Cloudflare bot-challenge pages before sanitising (scripts are stripped,
    # making the challenge invisible and leaving a blank white page).
    _CF_MARKERS = ("challenges.cloudflare.com", "_cf_chl_opt", "cf-chl-", "Just a moment...")
    if any(m in raw_html[:4096] for m in _CF_MARKERS):
        escaped_url = html.escape(source_url)
        escaped_display = html.escape(parsed.netloc or source_url)
        return HTMLResponse(
            (
                "<!DOCTYPE html><html><head><meta charset='utf-8'><title>Verification required</title>"
                "<style>"
                "html,body{margin:0;padding:0;height:100%;background:transparent;}"
                "body{display:flex;align-items:center;justify-content:center;"
                "font-family:Segoe UI,system-ui,Arial,sans-serif;}"
                ".wall{text-align:center;padding:2rem 1.5rem;max-width:340px;}"
                ".wall-icon{font-size:2.2rem;margin-bottom:.6rem;opacity:.6;}"
                ".wall-title{font-size:1rem;font-weight:600;margin:0 0 .4rem;}"
                ".wall-body{font-size:.85rem;color:#666;margin:0 0 1.2rem;}"
                ".wall-btn{display:inline-block;padding:.5rem 1.1rem;background:#1a73e8;"
                "color:#fff;text-decoration:none;border-radius:6px;"
                "font-size:.85rem;font-weight:500;}"
                ".wall-btn:hover{background:#1558b0;}"
                "</style></head>"
                "<body><div class='wall'>"
                "<div class='wall-icon'>🛡️</div>"
                "<div class='wall-title'>Bot verification required</div>"
                f"<div class='wall-body'>{escaped_display} uses Cloudflare bot protection "
                "which can't be passed in the embedded view.</div>"
                f"<a class='wall-btn' href='{escaped_url}' target='_blank' rel='noopener noreferrer'>"
                "Open in new tab</a>"
                "</div></body></html>"
            ),
            status_code=200,
        )

    sanitized = sanitize_source_html(raw_html)

    # Detect common paywall patterns before rendering, and return a clear
    # inline notice so the user knows to open in their browser.
    _PAYWALL_PATTERNS = re.compile(
        r"(?:"
        r"subscribe\s+to\s+(?:read|continue|access)"
        r"|subscriber[- ]only"
        r"|subscription\s+required"
        r"|create\s+a\s+(?:free\s+)?account\s+to\s+(?:read|continue)"
        r"|sign\s+in\s+to\s+(?:read|continue)"
        r"|this\s+article\s+is\s+(?:for\s+)?(?:subscribers|members)"
        r"|already\s+a\s+subscriber"
        r"|you(?:'ve|\s+have)\s+reached\s+your\s+(?:free\s+)?(?:article|story)\s+limit"
        r"|metered.paywall"
        r"|data-paywall"
        r")",
        re.IGNORECASE,
    )
    # Only check the first ~8 KB to avoid scanning whole articles.
    _probe_text = sanitized[:8192]
    if _PAYWALL_PATTERNS.search(_probe_text):
        escaped_url = html.escape(source_url)
        escaped_display = html.escape(parsed.netloc or source_url)
        return HTMLResponse(
            (
                "<!DOCTYPE html><html><head><meta charset='utf-8'><title>Subscription required</title>"
                "<style>"
                "html,body{margin:0;padding:0;height:100%;background:transparent;}"
                "body{display:flex;align-items:center;justify-content:center;"
                "font-family:Segoe UI,system-ui,Arial,sans-serif;}"
                ".wall{text-align:center;padding:2rem 1.5rem;max-width:340px;}"
                ".wall-icon{font-size:2.2rem;margin-bottom:.6rem;opacity:.6;}"
                ".wall-title{font-size:1rem;font-weight:600;margin:0 0 .4rem;}"
                ".wall-body{font-size:.85rem;color:#666;margin:0 0 1.2rem;}"
                ".wall-btn{"
                "display:inline-block;padding:.5rem 1.1rem;background:#1a73e8;"
                "color:#fff;text-decoration:none;border-radius:6px;"
                "font-size:.85rem;font-weight:500;}"
                ".wall-btn:hover{background:#1558b0;}"
                "</style></head>"
                "<body><div class='wall'>"
                "<div class='wall-icon'>🔒</div>"
                f"<div class='wall-title'>Subscription required</div>"
                f"<div class='wall-body'>{escaped_display} requires a subscription. "
                "Open in your browser to read with your account.</div>"
                f"<a class='wall-btn' href='{escaped_url}' target='_blank' rel='noopener noreferrer'>"
                "Open in new tab</a>"
                "</div></body></html>"
            ),
            status_code=200,
        )

    sanitized = normalize_proxy_lazy_media(sanitized)
    escaped_source = html.escape(str(response.url))
    proxy_style = (
        "<style>"
        "img[alt*='image unavailable' i],"
        "img[src*='grey-placeholder'],"
        "img[src*='placeholder']{display:none!important;}"
        "img[data-src],img[data-lazy-src],img[loading='lazy']{"
        "opacity:1!important;visibility:visible!important;filter:none!important;}"
        "#lectio-bar{position:fixed;bottom:0;left:0;right:0;z-index:2147483647;"
        "background:rgba(20,20,20,.82);color:#ccc;backdrop-filter:blur(4px);"
        "font:11px/1.5 system-ui,-apple-system,sans-serif;padding:5px 12px;"
        "display:flex;align-items:center;gap:8px;}"
        "#lectio-bar a{color:#7ec8f7;text-decoration:none;}"
        "#lectio-bar a:hover{text-decoration:underline;}"
        "#lectio-bar-close{margin-left:auto;cursor:pointer;opacity:.6;font-size:15px;line-height:1;padding:0 2px;}"
        "#lectio-bar-close:hover{opacity:1;}"
        "</style>"
    )
    proxy_bar = (
        f"<div id='lectio-bar'>"
        f"<span>Proxied view</span>"
        f"<span style='opacity:.4'>·</span>"
        f"<a href='{escaped_source}' target='_blank' rel='noopener noreferrer'>Open original ↗</a>"
        f"<span id='lectio-bar-close' onclick=\"document.getElementById('lectio-bar').remove()\" title='Dismiss'>×</span>"
        f"</div>"
    )

    # Ensure relative links and assets resolve against the fetched document URL.
    if "<head" in sanitized.lower():
        sanitized = re.sub(
            r"(<head\b[^>]*>)",
            r"\1" + f'<base href="{escaped_source}">{proxy_style}',
            sanitized,
            count=1,
            flags=re.IGNORECASE,
        )
    else:
        sanitized = (
            f'<!DOCTYPE html><html><head><base href="{escaped_source}">'
            f"<meta charset='utf-8'>{proxy_style}</head>"
            f"<body>{sanitized}</body></html>"
        )

    # Inject the proxy bar before </body> (or append if not found).
    if re.search(r"</body\s*>", sanitized, re.IGNORECASE):
        sanitized = re.sub(r"(</body\s*>)", proxy_bar + r"\1", sanitized, count=1, flags=re.IGNORECASE)
    else:
        sanitized += proxy_bar

    # Picker mode: outline links on hover and post the clicked link's href to the
    # parent (the Add-Feed modal), which derives a selector server-side. Page
    # scripts are already stripped by the sanitizer, so this is the only JS in
    # the frame. No navigation happens — every click is captured and cancelled.
    if picker:
        picker_assets = (
            "<style>"
            "html.lectio-picker,html.lectio-picker *{cursor:crosshair!important;}"
            ".lectio-pick-hover{outline:2px solid #1a73e8!important;"
            "outline-offset:1px!important;background:rgba(26,115,232,.08)!important;}"
            "#lectio-bar{display:none!important;}"
            "</style>"
            "<script>(function(){"
            "document.documentElement.classList.add('lectio-picker');"
            "var last=null;"
            "function nearestA(el){while(el&&el!==document.body){"
            "if(el.tagName==='A'&&el.getAttribute('href'))return el;el=el.parentElement;}return null;}"
            "document.addEventListener('mouseover',function(e){"
            "var a=nearestA(e.target);"
            "if(last&&last!==a)last.classList.remove('lectio-pick-hover');"
            "if(a)a.classList.add('lectio-pick-hover');last=a;},true);"
            "document.addEventListener('click',function(e){"
            "e.preventDefault();e.stopPropagation();"
            "var a=nearestA(e.target);"
            "if(a){try{parent.postMessage({type:'lectio-pick',href:a.href},location.origin);}catch(_){}}"
            "},true);"
            "document.addEventListener('submit',function(e){e.preventDefault();},true);"
            "})();</script>"
        )
        if re.search(r"</body\s*>", sanitized, re.IGNORECASE):
            sanitized = re.sub(r"(</body\s*>)", picker_assets + r"\1", sanitized, count=1, flags=re.IGNORECASE)
        else:
            sanitized += picker_assets

    return HTMLResponse(sanitized, status_code=200)


def is_probably_frame_blocked(headers: httpx.Headers) -> tuple[bool, str]:
    xfo = (headers.get("x-frame-options") or "").lower()
    if "deny" in xfo:
        return True, "x-frame-options=deny"
    if "sameorigin" in xfo:
        return True, "x-frame-options=sameorigin"

    csp = (headers.get("content-security-policy") or "").lower()
    if "frame-ancestors" in csp:
        match = re.search(r"frame-ancestors\s+([^;]+)", csp)
        if match:
            value = match.group(1).strip()
            if "*" not in value:
                return True, f"csp frame-ancestors={value}"
    return False, ""


def probe_frameability(source_url: str) -> dict[str, object]:
    parsed = urlparse(source_url)
    if parsed.scheme not in {"http", "https"}:
        return {"blocked": True, "reason": "unsupported-url-scheme"}

    try:
        source_url = url_guard.ensure_safe_outbound_url(source_url)
        with url_guard.build_client(timeout=8.0, headers={"User-Agent": READABILITY_USER_AGENT}) as client:
            with client.stream("GET", source_url) as response:
                blocked, reason = is_probably_frame_blocked(response.headers)
                final_url = str(response.url)
        return {
            "blocked": blocked,
            "reason": reason,
            "final_url": final_url,
        }
    except Exception as exc:
        # If probing fails, do not force-block; UI can still attempt direct load then fallback.
        return {
            "blocked": False,
            "reason": f"probe-failed: {exc}",
            "final_url": source_url,
        }


# Comment threads are never article content, but they defeat extraction: a
# blogspot/WordPress comments section carries hundreds of avatar/delete-icon
# images and huge text, so readability scores it above the actual post and the
# "more images = better content" fallback can't rescue it (the comic post-body
# has *fewer* images than the comment avatars). Stripping the section before
# extraction lets the real body win. Container-level selectors only — never a
# broad `[class*=comment]`, which would catch content like "commentary" or a
# "N comments" badge.
_COMMENT_SECTION_SELECTORS = (
    "#comments",
    ".comments",
    ".comments-area",
    ".comment-list",
    ".comment-thread",
    ".comment-section",
    "#comment-holder",
    "#disqus_thread",
    ".disqus",
    "#respond",
    ".comment-form",
    "#comment-form",
)


def _strip_comment_sections(raw_html: str) -> str:
    """Remove comment-thread containers from page HTML before extraction."""
    try:
        from bs4 import BeautifulSoup

        soup = BeautifulSoup(raw_html, "html.parser")
        removed = False
        for sel in _COMMENT_SECTION_SELECTORS:
            for el in soup.select(sel):
                el.decompose()
                removed = True
        return str(soup) if removed else raw_html
    except Exception:  # noqa: BLE001
        return raw_html


# In-body chrome that modern CMSes embed *inside* the article container, so it
# survives a content-selector match (Future plc's #article-body carries the
# social-share bar, follow/newsletter widgets, ad units and a related-posts
# aside among the tab figures). Removed before extraction so neither readability
# nor the selector/whole-body fallbacks drag it in. Kept to markers that are
# never article prose: named share/newsletter/ad/affiliate widgets, tooltips,
# and <aside> (tangential content by definition).
# A chrome match holding at least this share of the page's text is treated as
# the article itself and kept. Share bars and signup boxes run a few percent;
# an article container runs most of the page, so anything in between is safe.
_CHROME_MAX_TEXT_SHARE = 0.4

_ARTICLE_CHROME_SELECTORS = (
    '[class*="flexisites-social"]',  # Future plc social-share bar
    '[class*="byline-social"]',
    ".google-follow-us-button",
    '[class*="hawk-root"]',  # Future plc affiliate/product widget
    '[class*="social-share"]',
    '[class*="share-buttons"]',
    '[class*="sharing-buttons"]',
    '[class*="newsletter"]',
    '[id*="newsletter"]',
    '[class*="ad-unit"]',
    '[class*="tooltip"]',
    "aside",
)


def _strip_article_chrome(raw_html: str) -> str:
    """Remove in-body share/newsletter/ad/related chrome before extraction.

    See _ARTICLE_CHROME_SELECTORS — the containers a content selector would
    otherwise keep because the CMS nests them inside the article body itself."""
    try:
        from bs4 import BeautifulSoup

        soup = BeautifulSoup(raw_html, "html.parser")
        removed = False
        # Chrome is small next to the article it decorates. A match holding most
        # of the page's text is therefore not chrome — it's the article wearing
        # a word this list happens to look for. selfh.st's Self-Host Weekly is
        # a *newsletter*, so its body container matches [class*="newsletter"]
        # and stripping it took 99.9% of the page, leaving Readability view with
        # a lead image and nothing else.
        total_text = len(soup.get_text(" ", strip=True))
        for sel in _ARTICLE_CHROME_SELECTORS:
            for el in soup.select(sel):
                if total_text and len(el.get_text(" ", strip=True)) >= total_text * _CHROME_MAX_TEXT_SHARE:
                    LOGGER.info("[extract] keeping %s: holds most of the page's text, not chrome", sel)
                    continue
                el.decompose()
                removed = True
        # JW Player video carousels ("Latest Videos From … Watch full video
        # here:") sit in an unsemantic rounded card that also holds the header
        # bar and an invisible "watch here" link — no single class covers all
        # three. Climb from the player marker to that wrapper and drop the whole
        # card. Fires only where a real player exists, so non-video content is
        # untouched.
        for marker in soup.select('[class*="jwp-carousel"], [class*="aspect-video"], [class*="jw-player"]'):
            if marker.parent is None:
                continue  # already removed as part of an earlier card
            card = marker
            for anc in marker.parents:
                cls = " ".join(anc.get("class") or [])
                if "overflow-hidden" in cls and "rounded" in cls:
                    card = anc
                    break
                if anc.get("id") == "article-body" or anc.name in ("article", "main", "body"):
                    break
            card.decompose()
            removed = True
        return str(soup) if removed else raw_html
    except Exception:  # noqa: BLE001
        return raw_html


_LEAD_IMAGE_SKIP_RE = re.compile(r"(?:logo|icon|avatar|sprite|placeholder|default|blank)", re.IGNORECASE)


def _lead_image_from_html(raw_html: str, source_url: str) -> str | None:
    """The page's hero image from social-preview metadata (og:image, then
    twitter:image, then <link rel=image_src>).

    readability extracts the article *body*, but a site's hero/featured image
    lives in the page header outside it (Future plc's #article-body is all tab
    figures, no hero), so captures lost the lead image the list thumbnail and
    article view want. This recovers the URL the publisher itself nominates as
    the article's representative image. Obvious non-content (SVGs, logo/icon/
    placeholder names) is skipped so a site whose og:image is its logo doesn't
    get a logo stamped on every capture."""
    try:
        from bs4 import BeautifulSoup

        soup = BeautifulSoup(raw_html, "html.parser")
        url = ""
        for finder in (
            lambda: soup.find("meta", property="og:image"),
            lambda: soup.find("meta", attrs={"name": "twitter:image"}),
            lambda: soup.find("link", rel="image_src"),
        ):
            tag = finder()
            if tag:
                url = str(tag.get("content") or tag.get("href") or "").strip()
                if url:
                    break
        if not url or url.lower().rsplit("?", 1)[0].endswith(".svg"):
            return None
        if _LEAD_IMAGE_SKIP_RE.search(url.rsplit("/", 1)[-1]):
            return None
        return urljoin(source_url, url)
    except Exception:  # noqa: BLE001
        return None


_IMG_DIMENSION_SIGNATURE_RE = re.compile(r"_(\d{3,5}x\d{3,5})\.\w{3,4}(?:[?%]|$)")


def _image_dimension_signature(url: str) -> str | None:
    """A WxH token embedded in a CDN filename (e.g. Substack's own upload
    naming, ..._2884x1622.jpeg), used to recognize the same source photo
    re-served under a different re-encoded asset id. See the duplicate-hero
    guard in extract_readability_article."""
    m = _IMG_DIMENSION_SIGNATURE_RE.search(url)
    return m.group(1) if m else None


def _bs4_content_fallback(raw_html: str) -> str:
    """Extract article content via BS4 using known content-area selectors.

    Tries selectors in priority order and returns the cleaned HTML of the first
    matching element. Used when readability fails (too short) or strips images.
    """
    try:
        from bs4 import BeautifulSoup

        soup = BeautifulSoup(raw_html, "html.parser")
        for tag in soup.find_all(["nav", "header", "footer", "script", "style"]):
            tag.decompose()
        for selector_type, value in [
            ("id", "article-body"),  # Future plc (guitarplayer/guitarworld/musicradar/…)
            ("class", "body-description"),  # RebelMouse CMS (premierguitar.com) — raised
            # 2026-08-31: readability's own scoring locked
            # onto the page's header/hero block instead
            # (<article class="...image-article...">, matched
            # first by the "tag: article" fallback below,
            # before this selector was added), losing every
            # "Ex. N" tab-diagram image in the real lesson body.
            ("class", "post-body"),  # Blogger
            ("class", "entry-content"),  # WordPress / Blogger
            ("class", "post-content"),  # Ghost, common themes
            ("class", "article-body"),
            ("tag", "article"),
            ("tag", "main"),
        ]:
            if selector_type == "class":
                elem = soup.find(class_=lambda c, v=value: bool(c) and v in c.split())  # type: ignore[arg-type]
            elif selector_type == "id":
                elem = soup.find(id=value)
            else:
                elem = soup.find(value)
            if elem:
                for nav_div in elem.find_all("div", class_=lambda c: bool(c) and "nav" in c.split()):
                    nav_div.decompose()
                return str(elem)
        return ""
    except Exception:
        return ""


_BBCODE_SIGNAL_RE = re.compile(
    r"\[(?:b|i|u|s|strike|url|img|quote|code|list|color|size|h[1-6]|center|spoiler)[\]=\s/\]]",
    re.IGNORECASE,
)


def _looks_like_bbcode(text: str) -> bool:
    """True when text contains BBCode markup (at least two BBCode tag patterns).

    The old heuristic (bb > ht) rejected mixed-content feeds like Nexus Mods
    where <br> tags pushed the HTML count above the BBCode count even though
    the article body is primarily BBCode.  _bbcode_to_html() already handles
    mixed content safely (it skips html.escape when HTML tags are present).

    But genuine authored HTML — anything with block structure (<p>, <div>,
    headings, lists, tables) — is never BBCode, even when stray brackets trip the
    signal. Sphinx/Pelican math blogs (eli.thegreenplace.net) carry LaTeX alt text
    like "[I=\\int ...]" / "[s(x)=...]" that reads as a "[i]"/"[s]" tag. Treating
    such HTML as BBCode runs _bbcode_to_html's newline->\\<br\\> step over its
    newline-formatted source, shredding every paragraph into one line per break
    ("poem" layout). BBCode feeds use <br> for breaks, not these tags, so the guard
    is safe for them.
    """
    if re.search(r"<(?:p|div|h[1-6]|ul|ol|table|blockquote|section|article)\b", text, re.IGNORECASE):
        return False
    return len(_BBCODE_SIGNAL_RE.findall(text)) >= 2


def _safe_bb_url(raw: str) -> str:
    """Return raw only if it's http(s); block everything else."""
    stripped = raw.strip()
    return stripped if re.match(r"https?://", stripped, re.IGNORECASE) else "#"


# IPB/Invision's video tag (Nexus Mods news posts, e.g. "[youtube]CC_O_X6HLkI[/youtube]").
# Content is normally a bare 11-char video id, but accepts a full watch/share/embed
# URL too in case another BBCode dialect nests one. Anchored to the WHOLE trimmed
# tag body so a URL with extra junk after the id (a stray query param, trailing
# text) is rejected rather than mis-extracting a wrong id — same "never claim what
# can't be backed" rule the rest of this module follows.
_BB_YOUTUBE_RE = re.compile(r"\[youtube\](.*?)\[/youtube\]", re.IGNORECASE | re.DOTALL)
_BB_YOUTUBE_ID_RE = re.compile(
    r"^(?:https?://)?(?:(?:www\.)?youtube(?:-nocookie)?\.com/(?:embed/|watch\?(?:[^\s]*&)?v=|v/|shorts/)|youtu\.be/)?([A-Za-z0-9_-]{11})\s*$",
    re.IGNORECASE,
)


def _bbcode_to_html(text: str) -> str:
    """Convert common BBCode tags to HTML.

    Only HTML-escapes the source when it contains no existing HTML markup
    (i.e. pure BBCode text).  For mixed content (feedparser adds <br/> etc.)
    we leave the existing tags intact and only apply BBCode substitutions.
    """
    import html as _html

    if re.search(r"<[a-z!]", text, re.IGNORECASE):
        out = text  # already has HTML — don't double-escape
    else:
        out = _html.escape(text, quote=False)

    # block-level
    def _list_items(s: str) -> str:
        return re.sub(r"\[\*\]", "<li>", s, flags=re.IGNORECASE)

    out = re.sub(r"\[list=1\](.*?)\[/list\]", lambda m: "<ol>" + _list_items(m.group(1)) + "</ol>", out, flags=re.I | re.S)
    out = re.sub(r"\[list\](.*?)\[/list\]", lambda m: "<ul>" + _list_items(m.group(1)) + "</ul>", out, flags=re.I | re.S)
    out = re.sub(r"\[quote=([^\]]{1,100})\](.*?)\[/quote\]", r"<blockquote><cite>\1</cite>\2</blockquote>", out, flags=re.I | re.S)
    out = re.sub(r"\[quote\](.*?)\[/quote\]", r"<blockquote>\1</blockquote>", out, flags=re.I | re.S)
    out = re.sub(r"\[code\](.*?)\[/code\]", r"<pre><code>\1</code></pre>", out, flags=re.I | re.S)
    out = re.sub(r"\[center\](.*?)\[/center\]", r'<div style="text-align:center">\1</div>', out, flags=re.I | re.S)
    out = re.sub(r"\[spoiler(?:=[^\]]*)?\](.*?)\[/spoiler\]", r"<details><summary>Spoiler</summary>\1</details>", out, flags=re.I | re.S)
    out = re.sub(r"\[h([1-6])\](.*?)\[/h\1\]", r"<h\1>\2</h\1>", out, flags=re.I | re.S)
    # [line] — IPB/Invision's horizontal-rule tag (Nexus Mods news posts),
    # self-closing with no [/line] counterpart. Not in _BBCODE_SIGNAL_RE:
    # "line" alone would false-positive on real prose like matplotlib's
    # documented fmt string "[marker][line][color]" (freecodecamp.org),
    # confirmed live 2026-09-01 — genuine BBCode posts already carry other
    # signal tags ([size], [url], ...) that trigger detection without it.
    out = re.sub(r"\[line\]", r"<hr>", out, flags=re.I)

    # inline
    out = re.sub(r"\[b\](.*?)\[/b\]", r"<strong>\1</strong>", out, flags=re.I | re.S)
    out = re.sub(r"\[i\](.*?)\[/i\]", r"<em>\1</em>", out, flags=re.I | re.S)
    out = re.sub(r"\[u\](.*?)\[/u\]", r"<u>\1</u>", out, flags=re.I | re.S)
    out = re.sub(r"\[s\](.*?)\[/s\]", r"<s>\1</s>", out, flags=re.I | re.S)
    out = re.sub(r"\[strike\](.*?)\[/strike\]", r"<s>\1</s>", out, flags=re.I | re.S)
    out = re.sub(r"\[color=([^\]]{1,30})\](.*?)\[/color\]", r'<span style="color:\1">\2</span>', out, flags=re.I | re.S)
    # [size=1..7] maps to the HTML <font size> scale; larger values treated as px.
    _BB_SIZE_EM = {1: 0.6, 2: 0.75, 3: 1.0, 4: 1.15, 5: 1.4, 6: 2.0, 7: 3.0}

    def _bb_size(m: re.Match) -> str:
        n = int(m.group(1))
        css = f"{_BB_SIZE_EM[n]}em" if n in _BB_SIZE_EM else f"{n}px"
        return f'<span style="font-size:{css}">{m.group(2)}</span>'

    out = re.sub(r"\[size=(\d{1,3})\](.*?)\[/size\]", _bb_size, out, flags=re.I | re.S)

    # links / images — sanitize URLs
    def _url_tag(m: re.Match) -> str:
        href = _safe_bb_url(_html.unescape(m.group(1)))
        label = m.group(2).strip() or href
        return f'<a href="{_html.escape(href, quote=True)}" target="_blank" rel="noopener noreferrer">{label}</a>'

    out = re.sub(r"\[url=([^\]]{1,500})\](.*?)\[/url\]", _url_tag, out, flags=re.I | re.S)
    out = re.sub(
        r"\[url\](https?://[^\[]{1,500})\[/url\]",
        lambda m: (
            f'<a href="{_html.escape(_safe_bb_url(_html.unescape(m.group(1))), quote=True)}"'
            f' target="_blank" rel="noopener noreferrer">{_html.escape(m.group(1))}</a>'
        ),
        out,
        flags=re.I | re.S,
    )
    out = re.sub(
        r"\[img(?:=[^\]]*)?\](https?://[^\[]{1,500})\[/img\]",
        lambda m: f'<img src="{_html.escape(_safe_bb_url(_html.unescape(m.group(1))), quote=True)}" loading="lazy" style="max-width:100%">',
        out,
        flags=re.I | re.S,
    )

    def _bb_youtube(m: re.Match) -> str:
        id_match = _BB_YOUTUBE_ID_RE.match(_html.unescape(m.group(1)).strip())
        # Leave the tag untouched (not stripped) when the body doesn't parse as a
        # recognizable id/URL — same fallback the rest of this function uses for
        # anything it can't safely convert.
        return _youtube_embed_html(id_match.group(1)) if id_match else m.group(0)

    out = _BB_YOUTUBE_RE.sub(_bb_youtube, out)

    # newlines
    out = out.replace("\n", "<br>\n")
    return out


# Future plc's CDN spells a resize as -WIDTH-QUALITY (…/abc123-1280-80.jpg).
_FUTURE_SIZE_SUFFIX_RE = re.compile(r"-\d{3,5}-\d{2,3}(?=\.[A-Za-z0-9]+$)")


def _img_variant_key(url: str) -> str | None:
    """Host + folder + basename with a resize suffix removed, so two sizes of one picture compare equal."""
    try:
        parts = urlparse(html.unescape(url or ""))
    except ValueError:
        return None
    folder, _, name = parts.path.rpartition("/")
    stem = _FUTURE_SIZE_SUFFIX_RE.sub("", _IMG_SIZE_SUFFIX_RE.sub("", name))
    if not parts.netloc or not stem:
        return None
    return f"{parts.netloc.lower()}{folder}/{stem}".lower()


def _bs4_strip_opener(content_html: str, lead_image_url: str) -> str | None:
    """Remove the lead-image opener from content HTML using BeautifulSoup.

    Regex stripping leaves orphaned closing tags when the opener is nested inside
    a container that also holds sibling elements (e.g. Tumblr npf_row with multiple
    figures). BS4 removes the image and walks up to clean empty parent containers
    without breaking the surrounding HTML structure.

    Returns the stripped HTML string (may be empty), or None if the lead image
    URL was not found in the content (caller should not strip anything).
    """
    try:
        from bs4 import BeautifulSoup, NavigableString

        soup = BeautifulSoup(content_html, "html.parser")
        target_img: object = None
        for img in soup.find_all("img"):
            src = str(img.get("src") or "")
            if src == lead_image_url or html.unescape(src) == lead_image_url:
                target_img = img
                break
        if target_img is None:
            # Exact URL not found. Try Tumblr CDN size-variant matching:
            # URLs share https://64.media.tumblr.com/{media_hash}/{token}/ across sizes,
            # so a cached s1280x1920 lead can be matched to the s640x960 content image.
            _TUMBLR_CDN_PREFIX_RE = re.compile(r"^(https://64\.media\.tumblr\.com/[^/]+/[^/]+)/", re.IGNORECASE)
            _lead_prefix_m = _TUMBLR_CDN_PREFIX_RE.match(lead_image_url)
            if _lead_prefix_m:
                _lead_prefix = _lead_prefix_m.group(1) + "/"
                for img in soup.find_all("img"):
                    src = str(img.get("src") or "")
                    if src.startswith(_lead_prefix):
                        target_img = img
                        break
        if target_img is None:
            # Blogger CDN size-variant matching: URLs differ only in the size
            # segment before the filename.  Supports both simple size codes
            # (s320, s500, s1600) and complex crop codes (w1200-h630-p-k-no-nu).
            _BLOGGER_CDN_RE = re.compile(
                r"^(https://(?:\d+\.bp\.blogspot\.com|(?:blogger|lh\d+)\.googleusercontent\.com)/.+?)/[sw]\d+[^/]*/(.+)$",
                re.IGNORECASE,
            )
            _lead_blogger_m = _BLOGGER_CDN_RE.match(lead_image_url)
            if _lead_blogger_m:
                _lead_base = _lead_blogger_m.group(1)
                _lead_file = _lead_blogger_m.group(2)
                for img in soup.find_all("img"):
                    src = str(img.get("src") or "")
                    _src_m = _BLOGGER_CDN_RE.match(src)
                    if _src_m and _src_m.group(1) == _lead_base and _src_m.group(2) == _lead_file:
                        target_img = img
                        break
        if target_img is None:
            # Same picture at another size on the same host/folder: WordPress -WxH and Future's -WIDTH-QUALITY
            # (guitarworld lead …-1280-80.jpg vs body …-1200-80.jpg) — without this the hero showed twice.
            _lead_key = _img_variant_key(lead_image_url)
            if _lead_key:
                for img in soup.find_all("img"):
                    if _img_variant_key(str(img.get("src") or "")) == _lead_key:
                        target_img = img
                        break
        if target_img is None:
            return None  # Lead URL not in content — caller should clear lead_image_url
        node_to_remove = target_img
        while True:
            parent = node_to_remove.parent
            if parent is None or getattr(parent, "name", None) == "[document]":
                break
            meaningful = [c for c in parent.children if not (isinstance(c, NavigableString) and not str(c).strip())]
            if len(meaningful) == 1:
                node_to_remove = parent  # Container becomes empty — hoist removal
            else:
                break  # Siblings exist — remove only the current node
        node_to_remove.decompose()
        body = soup.body
        result = body.decode_contents().strip() if body else str(soup).strip()
        return result
    except Exception:
        return None


# Reader-view media CSS, shared by build_readability_response and
# _wrap_readability_html so the two paths can't diverge. iframes default to 16:9;
# the block below collects every exception (audio players that need a fixed
# height instead) in one place — add new embed-host overrides here.
_READER_VIEW_MEDIA_CSS = (
    "article img{max-width:100%;height:auto;max-height:240px;}article a>img{max-height:1.4em;vertical-align:middle;}"
    "article iframe{max-width:100%;width:100%;aspect-ratio:16/9;height:auto;border:0;}"
    # --- audio embeds: not 16:9, give each its own fixed height ---
    "p.lectio-embed{margin:1rem 0;}p.lectio-embed iframe[src*='spotify.com']{aspect-ratio:auto;height:152px;}"
    "article iframe[src*='bandcamp.com/EmbeddedPlayer']{aspect-ratio:auto;height:470px;}"
    "article iframe[src*='bandcamp.com/EmbeddedPlayer'][src*='size=small']{height:42px;}"
    "article iframe[src*='soundcloud.com']{aspect-ratio:auto;height:166px;}"
    # --- end audio embeds ---
    "article svg{width:1.2em;height:1.2em;vertical-align:middle;flex-shrink:0;}"
)


_MARKDOWN_URL_RE = re.compile(r"\.(?:md|markdown)(?:\.txt)?(?:$|[?#])", re.IGNORECASE)


def _is_markdown_response(content_type: str, url: str) -> bool:
    """True when a fetched document is Markdown, not HTML — either declared
    (text/markdown) or a Markdown file served as text/plain (Google's dev
    docs expose .md.txt variants of every page)."""
    ct = (content_type or "").split(";")[0].strip().lower()
    if ct == "text/markdown":
        return True
    return ct in {"text/plain", ""} and bool(_MARKDOWN_URL_RE.search(urlparse(url).path))


def markdown_to_article_html(md_text: str, source_url: str) -> tuple[str, str]:
    """Convert Markdown to sanitized article HTML; title = first heading."""
    import markdown as _markdown

    html_out = _markdown.markdown(
        md_text,
        extensions=["fenced_code", "tables", "sane_lists"],
    )
    title = source_url
    m = re.search(r"<h1[^>]*>(.*?)</h1>", html_out, re.IGNORECASE | re.DOTALL)
    if not m:
        m = re.search(r"<h2[^>]*>(.*?)</h2>", html_out, re.IGNORECASE | re.DOTALL)
    if m:
        title = re.sub(r"<[^>]+>", "", m.group(1)).strip() or source_url
    article_html = sanitize_readability_html(html_out).strip()
    article_html = _absolutize_article_urls(article_html, source_url)
    if not article_html:
        raise ValueError("No readable article content was found.")
    return title, article_html


# Statuses that mean "refused", not "absent" — the cue to retry once with a
# browser identity before believing the answer. Medium answers a *live* article
# with 404 or 403 depending on the URL when it doesn't like the client, and a
# 404 taken at face value told the user the article was gone and offered to
# delete it. Mirrors feed_discovery's escalation policy.
_READABILITY_REFUSAL_STATUSES = frozenset({401, 403, 404, 405, 410, 429, 451, 503})


# Publish-date sources in a page's head, ordered by how much publishers maintain
# them. <time> is LAST on purpose: a page has many and the first often belongs to a
# comment or a "latest posts" rail. Measured over the archive when the recovery
# script was written: article:published_time carried 1,811 of 2,030 hits.
_PUBDATE_SOURCES: tuple[tuple[str, re.Pattern[str]], ...] = (
    (
        "og:article:published_time",
        re.compile(r'<meta[^>]+(?:property|name)=["\']article:published_time["\'][^>]*content=["\']([^"\']+)', re.I),
    ),
    (
        "og:reversed-attr-order",
        re.compile(r'<meta[^>]+content=["\']([^"\']+)["\'][^>]*(?:property|name)=["\']article:published_time["\']', re.I),
    ),
    ("json-ld:datePublished", re.compile(r'"datePublished"\s*:\s*"([^"]+)"', re.I)),
    ("itemprop:datePublished", re.compile(r'<meta[^>]+itemprop=["\']datePublished["\'][^>]*content=["\']([^"\']+)', re.I)),
    ("meta:date", re.compile(r'<meta[^>]+name=["\'](?:date|pubdate|publish-date|DC\.date[^"\']*)["\'][^>]*content=["\']([^"\']+)', re.I)),
    ("time:datetime", re.compile(r'<time[^>]+datetime=["\']([^"\']+)', re.I)),
)


_WAYBACK_AVAILABILITY = "https://archive.org/wayback/available"
# Query parameters that identify a CAMPAIGN, not an article. Archives are keyed on
# the URL as crawled, so a feed's tracking suffix means "no snapshot" even when the
# article is archived — informit's ?p=2433607&WT.rss_f=…&WT.rss_a=… found nothing
# while the bare ?p=2433607 found it immediately.
_TRACKING_PARAM_RE = re.compile(
    r"^(?:utm_|wt\.|at_|mc_|pk_|ga_|hsa_|vero_|matomo_|piwik_|"
    r"fbclid|gclid|dclid|msclkid|igshid|mkt_tok|ncid|cmpid|cid|ref|referrer|source|spm)",
    re.IGNORECASE,
)


def strip_tracking_params(url: str) -> str:
    """Drop campaign/tracking query parameters, keeping the identifying ones.

    Deliberately a denylist, not a keeplist: for a URL like
    ``article.aspx?p=2433607`` the query IS the article's identity, so guessing at
    what to keep would throw the article away.
    """
    try:
        parts = urlsplit(url)
    except ValueError:
        return url
    if not parts.query:
        return url
    kept = [(k, v) for k, v in parse_qsl(parts.query, keep_blank_values=True) if not _TRACKING_PARAM_RE.match(k)]
    if len(kept) == len(parse_qsl(parts.query, keep_blank_values=True)):
        return url
    return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(kept, doseq=True), parts.fragment))


def wayback_snapshot_url(source_url: str) -> str | None:
    """The nearest Wayback snapshot of *source_url*, or None.

    One small JSON call to archive.org's availability API — no crawling, no
    scraping of the site that already refused us. This is the answer for an
    article whose publisher now serves a parked page or a section index over its
    own URL: the guard correctly refuses that page, and without a fallback the
    user is left with nothing better.
    """
    if not source_url or not source_url.startswith(("http://", "https://")):
        return None
    found = _wayback_lookup(source_url)
    if found:
        return found
    # Retry without tracking parameters: the archive was crawled from links that
    # carried none, so a feed's ?WT.rss_a=… suffix reads as an unarchived URL.
    cleaned = strip_tracking_params(source_url)
    return _wayback_lookup(cleaned) if cleaned != source_url else None


def _wayback_lookup(source_url: str) -> str | None:
    try:
        headers = {"User-Agent": READABILITY_USER_AGENT}
        with url_guard.build_client(timeout=10.0, headers=headers) as client:
            resp = url_guard.safe_get(
                client,
                f"{_WAYBACK_AVAILABILITY}?url={quote_plus(source_url)}",
                headers=headers,
            )
        resp.raise_for_status()
        snap = ((resp.json() or {}).get("archived_snapshots") or {}).get("closest") or {}
    except Exception:  # noqa: BLE001 — a missing fallback is not an error
        LOGGER.info("wayback lookup failed for %s", source_url, exc_info=True)
        return None
    if not snap.get("available"):
        return None
    url = str(snap.get("url") or "")
    return url or None


def mine_publish_date(raw_html: str | None) -> datetime | None:
    """The article's publish date from a page's own metadata, or None.

    Range-checked rather than trusted: a 1900 or 2099 value is a template
    placeholder, and a future date is a clock problem, not a publication.
    """
    if not raw_html:
        return None
    now = datetime.now(timezone.utc)
    for _name, pattern in _PUBDATE_SOURCES:
        for m in pattern.finditer(raw_html):
            raw = (m.group(1) or "").strip()
            dt = None
            try:
                dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
            except ValueError:
                ymd = re.match(r"(\d{4})-(\d{2})-(\d{2})", raw)
                if ymd:
                    try:
                        dt = datetime(int(ymd.group(1)), int(ymd.group(2)), int(ymd.group(3)), tzinfo=timezone.utc)
                    except ValueError:
                        dt = None
            if dt is None:
                continue
            if dt.tzinfo is None:
                dt = dt.replace(tzinfo=timezone.utc)
            if 1990 < dt.year and dt <= now + timedelta(days=2):
                return dt
    return None


def fetch_readability_article(
    source_url: str,
    *,
    capture: dict | None = None,
    max_tier: page_fetch.FetchTier = "proxy",
    ignore_cooldown: bool = False,
) -> tuple[str, str]:
    """Fetch *source_url* and return ``(title, article_html)``: the
    readability-extracted, sanitized article body. Shared by the reader-view
    route and save-article capture. Raises on fetch/extraction failure (a
    page_fetch.PageFetchError exposes .response/.status_code the same way an
    httpx.HTTPStatusError would, for callers that duck-type it — see
    services/saved_articles.py's dead-vs-blocked classification). Markdown
    documents (text/markdown, or .md/.md.txt paths served as plain text) are
    converted instead of readability-extracted.

    Escalates through services/page_fetch.py's honest -> browser -> proxy ->
    FlareSolverr ladder — not about winning an arms race (a host that blocks
    by IP still refuses honest and browser identity alike), but about not
    *mistaking a refusal for a deletion*: the caller flags 404/410 as "the
    article is gone", and a bot-wall answering 404 turned a live article into
    a delete prompt, which is also why _READABILITY_REFUSAL_STATUSES is wider
    than the default set. *max_tier* defaults to "proxy" (no FlareSolverr) —
    this runs on the synchronous reader-view render path too, where a ~55s
    FlareSolverr call would hang a page behind a spinner with no cancel; only
    callers that know they're off the request path (the re-fetch path) opt
    into "flaresolverr". *ignore_cooldown* is for a user-initiated re-fetch —
    a person clicking "Refetch content" is not the polite background traffic
    the cooldown exists to pace.

    *capture*, when given, receives the raw response body under ``"raw_html"``.
    readability strips head metadata, so a caller wanting the page's publish date
    would otherwise have to fetch it a second time."""
    result = page_fetcher.fetch(
        source_url,
        timeout=12.0,
        refusal_statuses=_READABILITY_REFUSAL_STATUSES,
        max_tier=max_tier,
        ignore_cooldown=ignore_cooldown,
    )
    if capture is not None:
        capture["raw_html"] = result.html
    if _is_markdown_response(result.headers.get("content-type", ""), source_url):
        return markdown_to_article_html(result.html, source_url)
    title, article_html = extract_readability_article(result.html, source_url)
    return title, _append_site_embeds(article_html, source_url, result.html)


# Below this much extracted article text, readability is treated as having
# failed — only then may the whole-body image rescue drag in the full page. A
# real article this short with no images is rare; the chrome-only image-heavy
# pages this guards against (Google Developers Blog) carry far more than this.
_WHOLE_BODY_RESCUE_MIN_TEXT = 1200


def _media_tag_count(html: str) -> int:
    """<img> + <iframe> tag count — the fallback-selection comparisons in
    extract_readability_article treat both as "real content", not just
    images. An allowlisted <iframe> embed (a soundslice.com tab player, a
    YouTube video) is exactly as much a reason to prefer one candidate over
    another as an <img> is; counting only <img> made a body-description
    selector match with real embedded content but zero inline images lose a
    0-vs-0 tie against readability's own chrome-only extraction, raised
    2026-08-31."""
    lowered = html.lower()
    return lowered.count("<img") + lowered.count("<iframe")


def extract_readability_article(raw_html: str, source_url: str) -> tuple[str, str]:
    """Readability-extract ``(title, article_html)`` from already-obtained page
    HTML — e.g. a rendered DOM captured by the user's browser (extension save),
    which sees past paywalls/bot-walls the server can't fetch through."""
    # Readability's clean_attributes strips width/height/style from its output
    # entirely, so style-only-sized images (NewsBlur's 18px glyph icons) blow
    # up to column width. Lift style px sizes onto attributes, capture every
    # image's size from the raw page, and reapply after extraction.
    raw_html = _strip_site_chrome(raw_html, source_url)
    # A site that names its article body gets that slice taken whole, as the source-article path does: readability's density scoring
    # drops a photo gallery's slides (little text) and keeps the page around them.
    content_selectors = site_content_plugins.content_selectors(source_url)
    sliced = _slice_to_content(raw_html, content_selectors) if content_selectors else None
    if sliced:
        # The slice has no <head>, so the title comes from the page it was cut from.
        title = _page_title_from_html(raw_html, source_url)
        return title, extract_full_page_article(sliced, source_url)[1]
    raw_html = _replace_youtube_facades(raw_html)
    raw_html = html_sanitize.lift_float_classes(raw_html)
    raw_html = html_sanitize.lift_img_style_sizes(raw_html)
    # Strip comment threads first — otherwise readability scores a big comments
    # section above the post and no image-count fallback can recover the body.
    raw_html = _strip_comment_sections(raw_html)
    # Then the in-body share/newsletter/ad/related chrome CMSes nest inside the
    # article container — otherwise a #article-body match keeps it (guitarplayer
    # led every capture with a share bar and a newsletter signup).
    raw_html = _strip_article_chrome(raw_html)
    _img_sizes = html_sanitize.collect_img_sizes(raw_html, base_url=source_url)
    doc = Document(raw_html, url=source_url)
    title = doc.short_title() or source_url
    summary = doc.summary(html_partial=True)
    summary = _reinject_readability_embeds(summary, raw_html)
    # Promote lazy-loading attributes BEFORE sanitizing. A page that ships
    # `<img data-lazy-src=…>` with no `src` (Wayback snapshots of WordPress
    # sites do exactly this) otherwise stores an image element with nothing to
    # load, which renders as an empty box. The helper already existed for the
    # web-view proxy; capture needs it more, because what it stores is what the
    # offline copy will forever contain.
    summary = normalize_proxy_lazy_media(summary)
    article_html = sanitize_readability_html(summary).strip()
    _bs4_fallback_used = False
    if len(article_html) < 300:
        # Readability found nothing meaningful (or just a short tagline/subtitle).
        # Lazy-attrs need the same pre-sanitize promotion as the primary summary
        # above, or a fallback selector's own lazy-loaded images (RebelMouse's
        # data-runner-src, etc.) lose their blank placeholder src to the
        # sanitizer with nothing promoted to replace it — raised 2026-08-31.
        fallback = normalize_proxy_lazy_media(_bs4_content_fallback(raw_html))
        if fallback:
            fallback_clean = sanitize_readability_html(fallback).strip()
            if fallback_clean and len(fallback_clean) > len(article_html):
                article_html = fallback_clean
                _bs4_fallback_used = True
    if not _bs4_fallback_used and "<img" in raw_html:
        # The trigger below stays <img>-only, deliberately: article_html at
        # this point already carries whatever _reinject_readability_embeds
        # recovered, which scans the WHOLE raw page for allowlisted <iframe>s
        # and bolts them onto readability's summary regardless of which text
        # container they came from — so a chrome-only extraction can look
        # "media-rich" once its embeds are recovered even though its actual
        # prose is garbage. Counting iframes here made needs_fallback go
        # false and skip looking for a better candidate at all. The
        # ACCEPTANCE comparisons just below are different: there an <iframe>
        # embed IS exactly as much a reason to prefer one full candidate over
        # another as an <img> is (raised 2026-08-31: a body-description
        # selector match with zero <img> but several soundslice.com tab
        # players lost a 0-vs-0 tie to readability's own chrome extraction,
        # so a page with real content fell through to whole-body-rescue and
        # dragged in nav chrome for no reason) — so those use _media_tag_count.
        raw_img_count = raw_html.lower().count("<img")
        art_img_count = article_html.lower().count("<img")
        # Fall back if readability stripped all images, or if the page is
        # image-heavy (>4 imgs) and readability kept fewer than half of them.
        needs_fallback = art_img_count == 0 or (raw_img_count > 4 and art_img_count < raw_img_count // 2)
        if needs_fallback:
            # Separate from art_img_count above (which stays <img>-only for the
            # trigger and the whole-body-rescue gate below): the two ACCEPTANCE
            # comparisons that follow need a combined-media baseline on both
            # sides, or a candidate with real iframe embeds but zero inline
            # images loses to article_html's own img-only count instead of its
            # actual (possibly iframe-recovered) media count — raised 2026-08-31.
            art_media_count = _media_tag_count(article_html)
            fallback = normalize_proxy_lazy_media(_bs4_content_fallback(raw_html))
            if fallback and _media_tag_count(fallback) > art_media_count:
                article_html = sanitize_readability_html(fallback).strip()
                art_img_count = article_html.lower().count("<img")
                art_media_count = _media_tag_count(article_html)
            # Last resort: readability *and* the selector fallback both kept
            # essentially no images on an image-heavy page — the catastrophic
            # case, not mere trimming. guitarplayer lessons are the example:
            # ~1 of ~54 images survive, and the dropped ones are the tablature
            # figures that *are* the lesson, on a DOM no content selector
            # matches. Take the whole body. Gated hard (≤1 kept, >10 present) so
            # a page where readability kept a reasonable share is never widened
            # into dragging in its chrome images.
            #
            # But only when readability failed to get the *text* either: a plain
            # text article with no images (Google Developers Blog) is complete as
            # readability returned it, and its page is "image-heavy" only from nav
            # and footer chrome — widening would replace the clean article with
            # the whole nav-laden page. So require readability's article to be
            # thin as well before reaching for the whole body.
            article_text_len = len(re.sub(r"<[^>]+>", "", article_html))
            if art_img_count <= 1 and raw_img_count > 10 and article_text_len < _WHOLE_BODY_RESCUE_MIN_TEXT:
                whole = _whole_body_content(raw_html)
                if whole and _media_tag_count(whole) > art_media_count:
                    article_html = whole
    # Prepend the publisher's hero image when the body doesn't already open with
    # it — the article's lead image lives in the page header, outside the content
    # readability extracts (a #article-body of tab figures has no hero).
    lead = _lead_image_from_html(raw_html, source_url)
    if lead:
        # A bare last-path-segment match used to stand in for "already
        # present," which false-positived on premierguitar.com's media-library
        # CDN (raised 2026-08-31): every image there is literally named
        # image.jpg, distinguished only by ?id=, so og:image's filename
        # matched ANY other image on the page and the real hero was silently
        # dropped as "already shown." _dedupe_img_src_key is the same
        # query-aware identity check _dedupe_readability_images uses, so the
        # two stay consistent.
        lead_key = _dedupe_img_src_key(lead)
        # readability hasn't absolutized the body's srcs yet (a featured image is often root-relative), while `lead` always is.
        # Comparing the raw srcs made the same hero show twice. The lookbehind skips data-src / data-lazy-src, which aren't rendered.
        body_srcs = [urljoin(source_url, html.unescape(src)) for src in re.findall(r'(?<![\w-])src="([^"]*)"', article_html)]
        # Substack (and likely other CDN-resize proxies) can serve og:image and
        # the post's own in-body header image as two DIFFERENT re-encoded asset
        # ids for the same source photo — raised 2026-08-31: og:image's id
        # didn't appear anywhere in the body, so the exact-match check below
        # missed it and prepended a genuine duplicate (same photo, twice).
        # Both asset filenames still carry the original upload's pixel
        # dimensions as a suffix (..._2884x1622.jpeg) even though the id
        # prefix differs, so fall back to that as a same-photo signal.
        lead_dim_sig = _image_dimension_signature(lead)
        lead_host = urlparse(lead).netloc
        already_present = (
            lead in article_html
            or any(_dedupe_img_src_key(src) == lead_key for src in body_srcs)
            or (
                lead_dim_sig is not None
                and any(
                    # Same host too, not dimensions alone (raised in review
                    # 2026-08-31): a CMS that resizes every hero to one standard
                    # preset (e.g. every image at 1200x630) would otherwise
                    # false-positive on any unrelated in-body image sharing that
                    # size, silently dropping a genuinely different hero. The
                    # motivating case (Substack) already shares a host between
                    # og:image and the in-body copy, so this doesn't lose the
                    # fix it was added for.
                    urlparse(src).netloc == lead_host and _image_dimension_signature(src) == lead_dim_sig
                    for src in body_srcs
                )
            )
        )
        if not already_present:
            article_html = f'<figure><img src="{html.escape(lead, quote=True)}"></figure>' + article_html
    article_html = _finalize_article_html(article_html, source_url, _img_sizes)
    if not article_html:
        raise ValueError("No readable article content was found.")
    return title, article_html


_WAYBACK_IMG_WRAPPER_RE = re.compile(
    r'(["\'])https?://web\.archive\.org/web/\d+[a-z]{0,3}_?/(https?://[^"\']+)\1',
    re.IGNORECASE,
)


def _unwrap_wayback_image_urls(article_html: str) -> str:
    """Rewrite a Wayback-snapshot-wrapped image URL back to the original it wraps.

    An archive-fallback capture (mode="archive", or the automatic mismatch/dead
    fallback in _refresh_captured_article_for_current_user) extracts content
    from a Wayback snapshot page, so every relative image src absolutizes
    against the *snapshot* URL — producing im_-wrapped web.archive.org URLs
    (http://web.archive.org/web/<ts>im_/<original>). That wrapper is far
    flakier than the site it wraps: raised 2026-08-31 against two live
    beehiiv-hosted entries where the im_ proxy 404'd while the exact same URL
    it wraps, fetched directly, returned 200 image/jpeg. The wrapper already
    carries the original URL, so unwrap it rather than storing the fragile
    proxy — nothing is gained by going through Wayback for the image once the
    HTML capture itself is done."""
    if "web.archive.org" not in article_html:
        return article_html
    return _WAYBACK_IMG_WRAPPER_RE.sub(r"\1\2\1", article_html)


def _finalize_article_html(article_html: str, source_url: str, img_sizes) -> str:
    """Shared post-processing for captured article HTML, whether readability-
    extracted or full-page. Reapplies image sizes captured from the raw page,
    resolves relative URLs against the source, unwraps any Wayback-snapshot
    image-proxy URL that resolution produced, and applies the entry pane's
    hotlink handling (route known hotlink hosts through /api/img; strip the
    Referer elsewhere so foreign-Referer placeholder hosts serve the real
    image)."""
    article_html = html_sanitize.reapply_img_sizes(article_html, img_sizes, base_url=source_url)
    article_html = _dedupe_readability_images(article_html)
    article_html = _strip_bandcamp_track_signature(article_html)
    article_html = _absolutize_article_urls(article_html, source_url)
    article_html = _unwrap_wayback_image_urls(article_html)
    article_html = proxy_hotlink_images(article_html)
    article_html = add_no_referrer_to_images(article_html)
    return article_html


def _page_title_from_html(raw_html: str, source_url: str) -> str:
    """Best page title without running readability's body extraction: og:title,
    then twitter:title, then <title>, then the first <h1>, then the URL."""
    try:
        from bs4 import BeautifulSoup

        soup = BeautifulSoup(raw_html, "html.parser")
        attr_options: tuple[dict[str, str], ...] = ({"property": "og:title"}, {"name": "twitter:title"})
        for attrs in attr_options:
            tag = soup.find("meta", attrs=cast(dict, attrs))
            if tag and str(tag.get("content", "")).strip():
                return str(tag["content"]).strip()
        if soup.title and soup.title.string and soup.title.string.strip():
            return soup.title.string.strip()
        h1 = soup.find("h1")
        if h1 and h1.get_text(strip=True):
            return h1.get_text(strip=True)
    except Exception:  # noqa: BLE001
        pass
    return source_url


def extract_full_page_article(raw_html: str, source_url: str) -> tuple[str, str]:
    """Capture the whole page body instead of readability-extracting it.

    The escape hatch for pages readability handles badly: a DocBook-style export
    whose prose is scattered across low-scoring sibling divs, or an article whose
    lead image sits in a text-free floated div that readability's cleaning drops
    (the Blood Meridian case). Those are structural extraction failures, not bad
    input — re-running readability just reproduces them, so this takes the whole
    <body> through the same sanitizer and post-processing instead.

    The tradeoff is deliberate and the opposite of readability's: on a blog-
    shaped page this keeps nav/sidebar/footer chrome, which readability would
    have stripped. It is for the document-shaped pages where that chrome is
    absent and the prose is the body. Only ``<script>``/``<style>``/``<nav>``/
    ``<header>``/``<footer>`` are removed, as obvious non-content that is never
    the article even on a document page."""
    raw_html = _strip_site_chrome(raw_html, source_url)
    raw_html = _replace_youtube_facades(raw_html)
    raw_html = html_sanitize.lift_float_classes(raw_html)
    raw_html = html_sanitize.lift_img_style_sizes(raw_html)
    img_sizes = html_sanitize.collect_img_sizes(raw_html, base_url=source_url)
    title = _page_title_from_html(raw_html, source_url)
    article_html = _whole_body_content(raw_html)
    article_html = _finalize_article_html(article_html, source_url, img_sizes)
    if not article_html:
        raise ValueError("The page had no usable content.")
    return title, article_html


def _whole_body_content(raw_html: str) -> str:
    """Sanitized whole-page body with only obvious non-content removed
    (script/style/nav/header/footer). Keeps everything readability would score
    away — the shared core of full-page capture and the image-rescue last resort
    in extract_readability_article."""
    body_html = raw_html
    try:
        from bs4 import BeautifulSoup

        soup = BeautifulSoup(raw_html, "html.parser")
        for tag in soup.find_all(["script", "style", "nav", "header", "footer"]):
            tag.decompose()
        # Comment threads are not article content, and a full-body capture would
        # otherwise swallow a huge comments section (the whole point of the
        # image-rescue was to keep content, not a wall of avatars).
        for sel in _COMMENT_SECTION_SELECTORS:
            for el in soup.select(sel):
                el.decompose()
        body = soup.body or soup
        body_html = str(body)
    except Exception:  # noqa: BLE001
        pass
    return html_sanitize.sanitize_html(body_html).strip()


def fetch_full_page_article(
    source_url: str,
    *,
    capture: dict | None = None,
    max_tier: page_fetch.FetchTier = "proxy",
    ignore_cooldown: bool = False,
) -> tuple[str, str]:
    """Fetch *source_url* and return ``(title, body_html)`` without readability
    extraction — see extract_full_page_article. Markdown is still converted.
    Shares fetch_readability_article's escalation ladder and *max_tier*/
    *ignore_cooldown* semantics (see its docstring) — this used to have no
    retry of any kind, so every caller now gets the honest->browser->proxy
    ladder for free, not just the FlareSolverr rung.

    *capture* receives the raw body under ``"raw_html"``, so a caller can mine the
    page's publish date from the fetch it already made."""
    result = page_fetcher.fetch(
        source_url,
        timeout=12.0,
        refusal_statuses=_READABILITY_REFUSAL_STATUSES,
        max_tier=max_tier,
        ignore_cooldown=ignore_cooldown,
    )
    if capture is not None:
        capture["raw_html"] = result.html
    if _is_markdown_response(result.headers.get("content-type", ""), source_url):
        return markdown_to_article_html(result.html, source_url)
    title, article_html = extract_full_page_article(result.html, source_url)
    return title, _append_site_embeds(article_html, source_url, result.html)


def build_readability_response(source_url: str) -> HTMLResponse:
    parsed = urlparse(source_url)
    if parsed.scheme not in {"http", "https"}:
        return HTMLResponse("<h1>Unsupported URL scheme.</h1>", status_code=400)

    try:
        title, article_html = fetch_readability_article(source_url)
    except Exception as exc:
        escaped_url = html.escape(source_url)
        escaped_error = html.escape(public_error(exc, "Fetching the article"))
        return HTMLResponse(
            (
                "<!DOCTYPE html><html><head><meta charset='utf-8'><title>Readability unavailable</title>"
                "<style>body{font-family:Segoe UI,Arial,sans-serif;margin:0;padding:1rem;line-height:1.45;}"
                "h1{font-size:1.05rem;margin:0 0 .5rem;}p{margin:.35rem 0;color:#555;}a{color:#0a5ca4;}</style></head>"
                f"<body><h1>Could not extract a readable article.</h1><p>{escaped_error}</p>"
                f"<p><a href='{escaped_url}' target='_blank' rel='noopener noreferrer'>Open original page</a></p></body></html>"
            ),
            status_code=200,
        )

    if saved_articles_service.looks_like_a_link_index(article_html):
        # A 200 does not mean the article is still there -- a URL folded into a
        # generic category/landing page (a retired blog's old post URLs all
        # redirecting to one hub) extracts as a list of unrelated links, not an
        # article. Found live 2026-09-17: guitarworld.com's retired
        # /lessons/<slug> URLs all now 301 to "Lessons Coverage | Guitar World",
        # and readability happily extracts that whole listing. Showing it is
        # strictly worse than an honest failure message.
        escaped_url = html.escape(source_url)
        return HTMLResponse(
            (
                "<!DOCTYPE html><html><head><meta charset='utf-8'><title>Readability unavailable</title>"
                "<style>body{font-family:Segoe UI,Arial,sans-serif;margin:0;padding:1rem;line-height:1.45;}"
                "h1{font-size:1.05rem;margin:0 0 .5rem;}p{margin:.35rem 0;color:#555;}a{color:#0a5ca4;}</style></head>"
                "<body><h1>Could not extract a readable article.</h1>"
                "<p>The page now at this URL looks like a list of links to other articles, not the article itself"
                " — it may have moved or the site may have reorganized.</p>"
                f"<p><a href='{escaped_url}' target='_blank' rel='noopener noreferrer'>Open original page</a></p></body></html>"
            ),
            status_code=200,
        )

    escaped_title = html.escape(title)
    escaped_source = html.escape(source_url)
    return HTMLResponse(
        (
            "<!DOCTYPE html><html><head><meta charset='utf-8'>"
            f"<title>{escaped_title}</title>"
            "<meta name='viewport' content='width=device-width, initial-scale=1'>"
            "<style>body{margin:0;background:#f6f8fb;color:#1a2430;font-family:Georgia,serif;}"
            "main{max-width:760px;margin:0 auto;padding:1.2rem 1rem 2rem;}"
            "header{font-family:Segoe UI,Arial,sans-serif;margin-bottom:1rem;padding-bottom:.75rem;border-bottom:1px solid #d4dbe5;}"
            "h1{margin:0;font-size:1.28rem;line-height:1.3;}"
            "a{color:#0a5ca4;}article{font-size:1.05rem;line-height:1.7;}" + _READER_VIEW_MEDIA_CSS + "article pre{white-space:pre-wrap;}"
            "article *{color:inherit !important;background-color:transparent !important;}"
            "</style></head>"
            f"<body><main><header><h1>{escaped_title}</h1>"
            f"<a href='{escaped_source}' target='_blank' rel='noopener noreferrer'>"
            "Open original</a></header>"
            f"<article>{article_html}</article></main></body></html>"
        ),
        status_code=200,
    )


# Blogger posts can ship an empty feed <title> while the real title lives only
# as the first body heading and in the post URL slug (e.g. treecardgames). Recover
# a readable title from the slug so the list/article don't show "(untitled)".
# Scoped to Blogger so genuinely-untitled posts elsewhere (e.g. Tumblr reblogs)
# keep their "(untitled)" label.
_BLOGGER_FEED_RE = re.compile(r"blogspot\.com|blogger\.com|/feeds/posts/default", re.IGNORECASE)


def _title_from_blogger_slug(link: str) -> str | None:
    """Humanize a Blogger post URL slug into a title, or None.

    e.g. ``https://x.blogspot.com/2026/06/gin-rummy-strategies-essential.html``
    → ``Gin Rummy Strategies Essential``.
    """
    if not link:
        return None
    slug = urlparse(link).path.rsplit("/", 1)[-1]
    slug = re.sub(r"\.html?$", "", slug, flags=re.IGNORECASE)
    words = [w for w in slug.split("-") if w]
    if not words:
        return None
    return " ".join(w.capitalize() for w in words)


def _decode_display_entities(text: str) -> str:
    """Fully decode HTML entities for plain-text display (e.g. titles).

    Titles are rendered as escaped text, so a stored literal entity shows raw
    ("Magus&rsquo; Castle"). Some feeds (notably Tumblr) double-encode
    ("&amp;rsquo;"), so unescape repeatedly until stable to surface the real
    character. Safe for display: the value is plain text that the template /
    JSON layer escapes again, so no markup can be introduced."""
    prev = text
    for _ in range(3):
        cur = html.unescape(prev)
        if cur == prev:
            break
        prev = cur
    return prev


def _display_title(entry) -> str:
    """Entry title for display, recovering Blogger empty-title posts from the slug.

    Returns the feed title when present; otherwise, for Blogger feeds only, a
    title humanized from the URL slug. Falls back to "" (template renders
    "(untitled)") for genuinely-untitled posts on other sites."""
    title = (getattr(entry, "title", None) or "").strip()
    if title:
        return _decode_display_entities(title)
    feed_url = str(getattr(entry, "feed_url", "") or "")
    link = str(getattr(entry, "link", "") or "")
    if _BLOGGER_FEED_RE.search(feed_url) or _BLOGGER_FEED_RE.search(link):
        return _title_from_blogger_slug(link) or ""
    return ""


# reader's FTS index is retired. Both search surfaces resolve in SQL
# (_search_entry_keys_in_sql for the Feeds view, _filter_star_keys_by_search for
# Kept/Saved) because search_entries builds a highlighted snippet per result,
# which measured at ~95% of a 10-20s search. Nothing queries the index, so it is
# no longer built or updated: that cost 1.3ms per new entry on every refresh and
# a file roughly the size of the reader DB (564MB against 743MB live).
# scripts/drop_search_index.py reclaims the space on an existing install.


def _filter_star_keys_by_search(keys: set[tuple[str, str]], search_terms: list[str]) -> set[tuple[str, str]] | None:
    """Narrow kept keys to those matching every search term, in SQL. None on
    any error → caller keeps the full set and post-filters in Python.

    Same technique as `_sorted_star_key_window`, for the same reason: joining
    the keys against the entries table is milliseconds where hydrating them via
    reader.get_entry is seconds. reader's own FTS index is deliberately *not*
    used here — `search_entries` builds a highlighted snippet per result, which
    measured at ~7.8ms/row (76s for one common term), so it costs far more than
    the scan it would replace.

    The predicate covers the Python haystack's fields (title, resolved feed
    title, link, author, summary) and adds the stored content, so a Saved
    search reaches the text of the article rather than only its metadata —
    the point of a read-later archive, and cheap here (~60ms over 11k keys)
    because it never leaves SQLite. Entry titles corrected via
    /entries/set-title are written into reader's own title column, so overrides
    are covered.

    Content is matched as stored, i.e. raw HTML, so a markup-ish term ("span",
    "http") matches nearly everything. Stripping tags would need a plain-text
    column maintained at ingest; not worth a schema change until a real search
    is hurt by it.
    """
    if not search_terms:
        return keys

    # LIKE wildcards in a user's search term are literals, not patterns.
    def _lit(term: str) -> str:
        return term.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")

    haystack = (
        "lower(COALESCE(e.title,'') || ' ' || COALESCE(f.user_title, f.title, '')"
        " || ' ' || COALESCE(e.feed,'') || ' ' || COALESCE(e.link,'')"
        " || ' ' || COALESCE(e.author,'') || ' ' || COALESCE(e.summary,'')"
        " || ' ' || COALESCE(e.content,''))"
    )
    term_sql = " AND ".join(f"{haystack} LIKE ? ESCAPE '\\'" for _ in search_terms)
    term_params = [f"%{_lit(t)}%" for t in search_terms]

    matched: set[tuple[str, str]] = set()
    try:
        conn = sqlite3.connect(str(tenancy.reader_db_path()), timeout=5.0)
        try:
            key_list = list(keys)
            # 2 bound vars per key; stay under SQLite's 999-variable limit.
            for i in range(0, len(key_list), 450):
                chunk = key_list[i : i + 450]
                values = ",".join("(?,?)" for _ in chunk)
                params = [v for pair in chunk for v in pair] + term_params
                rows = conn.execute(
                    f"WITH k(feed, id) AS (VALUES {values}) "
                    f"SELECT e.feed, e.id FROM entries e "
                    f"JOIN k ON e.feed = k.feed AND e.id = k.id "
                    f"LEFT JOIN feeds f ON f.url = e.feed "
                    f"WHERE {term_sql}",
                    params,
                ).fetchall()
                matched.update((str(r[0]), str(r[1])) for r in rows)
        finally:
            conn.close()
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("star key search query failed, post-filtering in Python: %s", exc)
        return None
    return matched


def _search_entry_keys_in_sql(
    search_terms: list[str],
    feed_urls: set[str],
    reader_read_filter: bool | None,
    limit: int,
) -> list[tuple[str, str]] | None:
    """Resolve a Feeds-view search in SQL. None on any error → caller falls
    back to the Python scan.

    The Feeds-view counterpart to `_filter_star_keys_by_search`, and it exists
    for the same reason that one avoids reader's FTS: `search_entries` builds a
    highlighted snippet per result. Measured on the live library (134k entries,
    2,888 feeds), snippet-building is ~95% of the cost and the query itself is
    almost free:

    | query  | FTS + hydrate | this |
    |--------|---------------|------|
    | python | 19.7s + 1.3s  | 1.6s |
    | guitar |  8.2s + 1.2s  | 1.1s |
    | coffee |  4.1s + 0.5s  | 1.3s |

    It also *finds more*: the haystack includes stored content, so a phrase from
    inside an article matches instead of only its metadata (coffee: 833 → 1,237).
    That is the same behavior Saved search already has, so the two surfaces now
    agree — previously one searched article text and the other didn't.

    Same known tradeoff as the Saved path, inherited deliberately: content is
    matched as stored, i.e. raw HTML, so a markup-ish term ("span", "http")
    matches nearly everything. Fixing that needs a plain-text column maintained
    at ingest; not worth a schema change until a real search is hurt.
    """
    if not search_terms:
        return None

    def _lit(term: str) -> str:
        return term.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")

    haystack = (
        "lower(COALESCE(e.title,'') || ' ' || COALESCE(f.user_title, f.title, '')"
        " || ' ' || COALESCE(e.feed,'') || ' ' || COALESCE(e.link,'')"
        " || ' ' || COALESCE(e.author,'') || ' ' || COALESCE(e.summary,'')"
        " || ' ' || COALESCE(e.content,''))"
    )
    where = [f"{haystack} LIKE ? ESCAPE '\\'" for _ in search_terms]
    params: list = [f"%{_lit(t)}%" for t in search_terms]

    read_clause = {
        True: "e.read = 1",
        False: "(e.read IS NULL OR e.read != 1)",
    }.get(reader_read_filter)
    if read_clause:
        where.append(read_clause)

    # Scope to the selected feeds in SQL when they fit under SQLite's
    # 999-variable limit, so LIMIT applies to rows the user can actually see.
    # Above that, match unscoped and let the caller drop out-of-scope feeds —
    # the same shape the FTS path had, and the same under-fill caveat.
    scoped = len(feed_urls) <= 900
    if scoped:
        placeholders = ",".join("?" for _ in feed_urls)
        where.append(f"e.feed IN ({placeholders})")
        params.extend(feed_urls)

    params.append(limit)
    try:
        conn = sqlite3.connect(f"file:{tenancy.reader_db_path()}?mode=ro", uri=True, timeout=10.0)
        try:
            rows = conn.execute(
                "SELECT e.feed, e.id FROM entries e "
                "LEFT JOIN feeds f ON f.url = e.feed "
                f"WHERE {' AND '.join(where)} "
                "ORDER BY COALESCE(e.published, e.first_updated) DESC LIMIT ?",
                params,
            ).fetchall()
        finally:
            conn.close()
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("[search] SQL search failed, falling back to scan: %s", exc)
        return None
    return [(str(r[0]), str(r[1])) for r in rows]


def _sorted_star_key_window(
    keys: set[tuple[str, str]],
    *,
    sort_by: str,
    sort_dir: str,
    reader_read_filter: bool | None,
    limit: int,
) -> list[tuple[str, str]]:
    """Sort + read-filter + clip starred keys in SQL, returning only the keys
    for the visible window. At a few thousand saves, hydrating every key via
    reader.get_entry takes seconds; joining the keys against the entries table
    for just the sort value is milliseconds. Keys whose entry no longer exists
    don't join (same outcome as get_entry -> None). Falls back to the full key
    set on any SQL error."""
    # "starred" orders by saved_entries.saved_at, which lives in the META DB —
    # this query only joins the reader DB, so there is no column here to sort
    # by. Selecting a constant keeps the read-filter and existence join doing
    # their work, and the star dates are merged in below.
    #
    # This mattered the moment the Saved Inbox started using the star order: the
    # window is only computed when the star set exceeds the fetch limit, which
    # on any real backlog is always — so falling through to `e.first_updated`
    # here silently picked the visible page by RECEIVED date and then sorted
    # that arbitrary page. The unchunked case looked fine, which is exactly what
    # made it worth checking the chunked one.
    by_star = sort_by == "starred"
    by_size = sort_by == "size"
    sort_expr = (
        "''" if by_star or by_size else "COALESCE(e.published, e.updated, e.first_updated)" if sort_by == "post" else "e.first_updated"
    )
    read_clause = {
        True: " AND e.read = 1",
        False: " AND (e.read IS NULL OR e.read != 1)",
    }.get(reader_read_filter, "")
    scored: list[tuple[str, str, str]] = []
    try:
        conn = sqlite3.connect(str(tenancy.reader_db_path()), timeout=5.0)
        try:
            key_list = list(keys)
            # 2 bound vars per key; stay under SQLite's 999-variable limit.
            for i in range(0, len(key_list), 450):
                chunk = key_list[i : i + 450]
                values = ",".join("(?,?)" for _ in chunk)
                params = [v for pair in chunk for v in pair]
                rows = conn.execute(
                    f"WITH k(feed, id) AS (VALUES {values}) "
                    f"SELECT e.feed, e.id, {sort_expr} AS sv "
                    f"FROM entries e JOIN k ON e.feed = k.feed AND e.id = k.id"
                    f" WHERE 1=1{read_clause}",
                    params,
                ).fetchall()
                scored.extend((str(r[0]), str(r[1]), str(r[2] or "")) for r in rows)
        finally:
            conn.close()
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("star key window query failed, hydrating all keys: %s", exc)
        return list(keys)
    if by_star:
        # Merge the star dates from the meta DB. A key with no row cannot really
        # happen (the set came from saved_entries) but sorts last rather than
        # raising, so a race during an unstar degrades to a misplaced row.
        try:
            with get_meta_connection() as _mc:
                _when = {
                    (str(r[0]), str(r[1])): str(r[2] or "") for r in _mc.execute("SELECT feed_url, entry_id, saved_at FROM saved_entries")
                }
        except Exception as exc:  # noqa: BLE001
            LOGGER.warning("star date lookup failed, keeping key order: %s", exc)
            _when = {}
        scored = [(f, i, _when.get((f, i), "")) for f, i, _ in scored]
    elif by_size:
        # Merge sizes from the starred-archive DB (a third DB from this
        # function's usual two). Zero-padded to a fixed width so the shared
        # string sort below still orders numerically. An entry with no
        # completed archive (never captured, or still pending) has no size
        # yet — sorts as smallest, which is correct either way: it is not
        # one of the heavy items this sort exists to surface.
        try:
            with archive_conn() as _ac:
                _size = {
                    (str(r[0]), str(r[1])): f"{int(r[2] or 0):020d}"
                    for r in _ac.execute(
                        "SELECT feed_url, entry_id, content_size_bytes FROM archived_entry WHERE content_size_bytes IS NOT NULL"
                    )
                }
        except Exception as exc:  # noqa: BLE001
            LOGGER.warning("archive size lookup failed, keeping key order: %s", exc)
            _size = {}
        scored = [(f, i, _size.get((f, i), "00000000000000000000")) for f, i, _ in scored]
    scored.sort(key=lambda t: t[2], reverse=(sort_dir != "asc"))
    return [(f, i) for f, i, _sv in scored[:limit]]


@dataclass(frozen=True, slots=True)
class _LightEnclosure:
    href: str | None


@dataclass(frozen=True, slots=True)
class _LightEntry:
    """Minimal stand-in for a `reader.Entry`, built straight from a SQL row —
    only the fields `list_entries_for_feeds`'s light-record loop reads.
    `_light_entries_from_sql` uses this to skip reader's own per-row
    hydration for enrich=False callers, where none of what hydration builds
    (Content/Author objects, a `feeds` join) ever gets used. See
    docs/architecture/views.md.

    No `feed` attribute on purpose: the enrichment phase checks `hasattr(entry,
    "feed")`, and if that ever ran against a leaked shim it must fail loudly
    (AttributeError) rather than silently render wrong data — the same reason
    fields only the search/tag/site-filter branches read (feed_resolved_title,
    summary, authors_str, resource_id) are also omitted: those branches are
    excluded from the gate that makes `_light_entries_from_sql` run at all, so
    reaching them here is a gating bug, and should surface as one."""

    feed_url: str
    id: str
    title: str | None
    link: str | None
    published: datetime | None
    updated: datetime | None
    added: datetime | None
    read: bool
    read_modified: datetime | None
    added_by: str
    enclosures: tuple[_LightEnclosure, ...] = ()


def _reader_ts(value: str | None) -> datetime | None:
    """Parse a reader-DB timestamp column the same way reader's own
    `convert_timestamp` does: naive-UTC `isoformat(" ")`, tagged UTC on read."""
    if not value:
        return None
    return datetime.fromisoformat(value).replace(tzinfo=timezone.utc)


def _enclosures_for_keys(conn: sqlite3.Connection, keys: list[tuple[str, str]]) -> dict[tuple[str, str], tuple[_LightEnclosure, ...]]:
    """Enclosures for specific (feed, id) keys, parsed from reader's stored
    JSON. Used only to back-fill link-less entries (Buzzsprout podcast feeds
    ship no <link>, only an audio enclosure — see `_derived_entry_link`); a
    second narrow query for just those rows beats widening every row's SELECT
    with a variable-length JSON column nothing else needs."""
    result: dict[tuple[str, str], tuple[_LightEnclosure, ...]] = {}
    for i in range(0, len(keys), 450):
        chunk = keys[i : i + 450]
        values = ",".join("(?,?)" for _ in chunk)
        params = [v for pair in chunk for v in pair]
        rows = conn.execute(
            f"SELECT feed, id, enclosures FROM entries WHERE (feed, id) IN (VALUES {values})",
            params,
        ).fetchall()
        for row in rows:
            if not row["enclosures"]:
                continue
            try:
                items = json.loads(row["enclosures"])
            except Exception:  # noqa: BLE001
                continue
            result[(str(row["feed"]), str(row["id"]))] = tuple(_LightEnclosure(href=d.get("href")) for d in items)
    return result


def _light_entries_from_sql(
    feed_urls: set[str],
    *,
    reader_read_filter: bool | None,
    sort_by: str,
    sort_dir: str,
    limit: int,
) -> list[_LightEntry] | None:
    """Fetch `list_entries_for_feeds`' light_records straight from SQL instead
    of through reader.get_entries()/get_entry(), for enrich=False callers with
    no tag filter, search, star_only, archived filter, or history sort (see
    the gate at this function's one call site). reader's own hydration
    JSON-decodes content/enclosures/author and joins against `feeds` per row —
    all irrelevant work for a caller that only wants
    feed_url/id/title/link/dates/read.

    Measured live: a feed whose entries carry heavy embedded content took
    ~1ms/entry to hydrate this way (7s for 7,153 entries, in a folder whose
    "Read Above" fallback — anchored on an already-read entry, so it has to
    scan the whole folder, not just the unread slice — took 9s total). This
    path is the fix for that: no Entry object is ever built.

    Returns None on any error so the caller falls back to the hydrated path —
    same "degrade, never break" contract as `_sorted_star_key_window`."""
    sort_col = _ENTRY_SORT_SQL if sort_by == "post" else "first_updated"
    read_clause = {
        True: " AND entries.read = 1",
        False: " AND (entries.read IS NULL OR entries.read != 1)",
    }.get(reader_read_filter, "")
    order = "ASC" if sort_dir == "asc" else "DESC"
    try:
        conn = sqlite3.connect(str(tenancy.reader_db_path()), timeout=5.0)
        conn.row_factory = sqlite3.Row
        try:
            feed_list = list(feed_urls)
            rows: list[sqlite3.Row] = []
            for i in range(0, len(feed_list), 999):
                chunk = feed_list[i : i + 999]
                placeholders = ",".join("?" for _ in chunk)
                rows.extend(
                    conn.execute(
                        f"SELECT feed, id, title, link, published, updated, first_updated,"
                        f" read, read_modified, added_by, {sort_col} AS sort_val"
                        f" FROM entries WHERE feed IN ({placeholders}){read_clause}"
                        f" ORDER BY {sort_col} {order} LIMIT ?",
                        [*chunk, limit],
                    ).fetchall()
                )
            # Harmless no-op for the (near-universal) single-chunk case; needed
            # for >999 feeds, whose per-chunk LIMITs must be re-applied globally
            # after merging — same reasoning as the existing ASC/DESC SQL paths.
            rows.sort(key=lambda r: r["sort_val"] or "", reverse=(order == "DESC"))
            rows = rows[:limit]

            linkless_keys = [(str(r["feed"]), str(r["id"])) for r in rows if not r["link"]]
            enclosures_by_key = _enclosures_for_keys(conn, linkless_keys) if linkless_keys else {}

            entries = [
                _LightEntry(
                    feed_url=str(row["feed"]),
                    id=str(row["id"]),
                    title=row["title"],
                    link=row["link"],
                    published=_reader_ts(row["published"]),
                    updated=_reader_ts(row["updated"]),
                    added=_reader_ts(row["first_updated"]),
                    read=row["read"] == 1,
                    read_modified=_reader_ts(row["read_modified"]),
                    added_by=row["added_by"] or "",
                    enclosures=enclosures_by_key.get((str(row["feed"]), str(row["id"])), ()),
                )
                for row in rows
            ]
        finally:
            conn.close()
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("light entry fetch failed, falling back to hydration: %s", exc)
        return None
    return entries


def _entry_link_site_host(entry) -> str:
    """The bare host of an entry's link — no userinfo/port, leading www. folded."""
    net = urlparse(str(getattr(entry, "link", "") or "")).netloc.split("@")[-1].split(":")[0].lower()
    return net[4:] if net.startswith("www.") else net


def _split_site_terms(terms: list[str]) -> tuple[list[str], list[str]]:
    """Pull ``site:<host>`` tokens out of a term list → (regular_terms, hosts).

    ``site:`` scopes a search to entries whose *link host* is that host or a
    subdomain of it — precise where a bare host term also matches the host
    appearing in an article's body or in another feed's post. Used by the
    File-Saved "review this host" link so a multi-feed host's unfiled saves can
    be listed exactly."""
    regular: list[str] = []
    sites: list[str] = []
    for tok in terms:
        if tok.startswith("site:") and len(tok) > 5:
            host = tok[5:].lstrip("@").split("/")[0]
            if host.startswith("www."):
                host = host[4:]
            if host:
                sites.append(host)
        else:
            regular.append(tok)
    return regular, sites


# Escalating fetch-window multiples list_entries_for_feeds retries at when
# hide_locked_comics/hide_unpremiered leave a page short -- see that
# function's own docstring for why these two specific multiples.
_HIDE_FILTER_UNDERFILL_RETRY_LIMIT_MULTIPLIERS = (2, 4, 8)
# Callers asking for a real page (default 250, rarely more than a few
# thousand) are exactly what the retry helps -- but _resolve_view_posts /
# mark_entries_range_read pass 1,000,000 as a "no real limit" sentinel, and
# for those the retry's whole premise (a fetch capped at `limit` dropped rows
# a bigger `limit` might recover) is backwards: the fetch was never bound by
# that number in the first place (no Lectio library has anywhere near a
# million entries in one scope), so `len(result) < limit` is guaranteed True
# and every retry attempt repeats the SAME full-cost fetch for nothing --
# measured live, 4x the cost of the original call, on every "All Feeds"-scale
# whole-view resolution once any feed anywhere had hide_locked_comics on.
# Anything above this ceiling skips the retry entirely.
_HIDE_FILTER_UNDERFILL_RETRY_MAX_LIMIT = 10_000


def list_entries_for_feeds(
    feed_urls: set[str],
    limit: int = 250,
    sort_by: str = "post",
    sort_dir: str = "asc",
    read_filter: str = "all",
    star_only: bool = False,
    selected_tag: str | None = None,
    selected_feed_tag: str | None = None,
    search_query: str | None = None,
    kept_scope: str = "kept",
    archived: bool | None = None,
    enrich: bool = True,
    min_fill: float = 1.0,
) -> list[dict]:
    """Thin retry wrapper around `_list_entries_for_feeds_fetch`.

    That function fetches only `limit` rows from reader before its per-entry
    hide_locked_comics/hide_unpremiered filter runs (see its own comments) --
    a locked/unpremiered entry occupying part of that window is dropped with
    nothing behind it to backfill the slot, so a page can render shorter than
    `limit` even though older, unhidden entries exist further back
    (Plan.md, "hide_locked_comics/hide_unpremiered can under-fill a page").

    Retrying with a larger fetch window fixes it without having to push the
    predicate into every one of that function's several SQL/reader fetch
    strategies -- `hide_unpremiered` in particular depends on a live-status
    cache keyed off a video id parsed from the entry's own link, not a plain
    meta-DB column a query could join against.

    Only attempted when one of those filters could actually be active for
    this scope (the common case pays one extra `len(result) >= limit` check
    and nothing else) and bounded by
    `_HIDE_FILTER_UNDERFILL_RETRY_LIMIT_MULTIPLIERS` -- there is no reliable
    way to tell "upstream is exhausted" from "the filter is dropping rows"
    using only this function's own return value, so a fully locked/unpremiered
    feed costs a handful of bounded retry fetches rather than one, not an
    early exit.

    Also skipped outright above `_HIDE_FILTER_UNDERFILL_RETRY_MAX_LIMIT`: a
    caller passing `limit=1_000_000` as a "give me everything" sentinel
    (`_resolve_view_posts`, `mark_entries_range_read`) can never satisfy
    `len(result) >= limit`, so without this guard the retry fired on every
    such call whenever a hide filter was active anywhere in scope, at full
    fetch cost each attempt, for a fetch that was never actually window-bound
    to begin with -- see that constant's own comment.
    """
    _stats: dict = {}
    result = _list_entries_for_feeds_fetch(
        feed_urls,
        limit=limit,
        sort_by=sort_by,
        sort_dir=sort_dir,
        read_filter=read_filter,
        star_only=star_only,
        selected_tag=selected_tag,
        selected_feed_tag=selected_feed_tag,
        search_query=search_query,
        kept_scope=kept_scope,
        archived=archived,
        enrich=enrich,
        _stats=_stats,
    )
    # min_fill < 1 accepts a slightly short page without the retry: a hide filter dropping one row of 250 otherwise
    # re-ran the whole fetch (~1.5s on a big folder) to backfill a row nobody sees. Only callers that never depend
    # on a full page opt in -- a delta chunk fetch must not, or it could come back with nothing new.
    if len(result) >= limit * min_fill or not feed_urls or limit > _HIDE_FILTER_UNDERFILL_RETRY_MAX_LIMIT:
        return result
    if _stats.get("exhausted"):
        # The raw fetch came back short of its window, so nothing older exists to backfill with. Without this,
        # any folder with fewer matches than `limit` (every small unread folder) re-ran the whole fetch up to
        # three more times whenever a hide filter was on anywhere in scope.
        return result

    with get_meta_connection() as _prefs_conn:
        _prefs_for_gate = get_all_feed_display_prefs(_prefs_conn)
    _hide_filter_could_apply = (
        hide_locked_comics_global()
        or youtube_hide_unpremiered_global()
        or any(
            _prefs_for_gate.get(fu, _DISPLAY_PREF_DEFAULTS).get("hide_locked_comics")
            or _prefs_for_gate.get(fu, _DISPLAY_PREF_DEFAULTS).get("hide_unpremiered")
            for fu in feed_urls
        )
    )
    if not _hide_filter_could_apply:
        return result

    for _multiplier in _HIDE_FILTER_UNDERFILL_RETRY_LIMIT_MULTIPLIERS:
        _attempt_limit = limit * _multiplier
        bigger = _list_entries_for_feeds_fetch(
            feed_urls,
            limit=_attempt_limit,
            sort_by=sort_by,
            sort_dir=sort_dir,
            read_filter=read_filter,
            star_only=star_only,
            selected_tag=selected_tag,
            selected_feed_tag=selected_feed_tag,
            search_query=search_query,
            kept_scope=kept_scope,
            archived=archived,
            enrich=enrich,
        )
        if len(bigger) > len(result):
            result = bigger
        if len(result) >= limit:
            break
        # No early exit otherwise: `bigger` being short of `_attempt_limit` does
        # NOT mean the raw fetch upstream is exhausted -- it means the filter
        # dropped rows, which is exactly the case being retried for. Only the
        # fixed multiplier list above bounds the worst case (an entirely
        # locked/unpremiered feed), not an "upstream is empty" inference.
    return result[:limit]


def _list_entries_for_feeds_fetch(
    feed_urls: set[str],
    limit: int = 250,
    sort_by: str = "post",
    sort_dir: str = "asc",
    read_filter: str = "all",
    star_only: bool = False,
    selected_tag: str | None = None,
    selected_feed_tag: str | None = None,
    search_query: str | None = None,
    kept_scope: str = "kept",
    archived: bool | None = None,
    enrich: bool = True,
    _stats: dict | None = None,
) -> list[dict]:
    entries: list[dict] = []
    if not feed_urls:
        return entries

    normalized_sort_by = normalize_sort_by(sort_by, allow_starred=True)
    normalized_sort_dir = normalize_sort_dir(sort_dir)
    normalized_read_filter = normalize_read_filter(read_filter)
    normalized_star_only = normalize_star_only(star_only)
    normalized_selected_tag = normalize_tag_value(selected_tag)
    normalized_selected_feed_tag = normalize_tag_value(selected_feed_tag)
    search_terms = search_terms_from_query(search_query)
    # site:<host> narrows to entries on that link host; it never goes to the
    # text-matching paths (SQL or haystack), only the link-host filter below.
    search_terms, site_hosts = _split_site_terms(search_terms)

    reader_read_filter: bool | None = None
    if normalized_read_filter == "unread":
        # Composes with star_only: the Saved view can narrow to unread.
        reader_read_filter = False
    elif normalized_read_filter == "history" and not normalized_star_only:
        reader_read_filter = True

    # Fetch app-managed entry metadata only for feeds in the active view.
    saved_entries_set: set[tuple[str, str]] = set()
    saved_at_map: dict[tuple[str, str], datetime] = {}
    read_state_map: dict[tuple[str, str], datetime] = {}
    feed_url_values = tuple(feed_urls)
    placeholders = ",".join("?" for _ in feed_url_values)
    history_fast_keys: list[tuple[str, str]] = []
    with get_meta_connection() as conn:
        rows = conn.execute(
            f"SELECT feed_url, entry_id, saved_at FROM saved_entries WHERE feed_url IN ({placeholders})",
            feed_url_values,
        ).fetchall()
        saved_entries_set = {(row["feed_url"], row["entry_id"]) for row in rows}
        # saved_at is written in two shapes — SQLite's CURRENT_TIMESTAMP
        # ("2026-07-29 04:28:19") and ISO-8601 from imports/migrations
        # ("2026-07-29T04:28:19+00:00"). Parsing rather than string-sorting is
        # what keeps a single day's stars in the right order.
        for row in rows:
            dt = _parse_stored_dt(row["saved_at"])
            if dt is not None:
                saved_at_map[(row["feed_url"], row["entry_id"])] = dt
        if normalized_read_filter == "history" and not normalized_star_only:
            # Fast path: read_history table stores recent individually-read entries
            # in read order with timestamps. Avoids scanning all read entries in
            # reader's DB when the user has been using the app with this version.
            hist_rows = conn.execute(
                f"SELECT feed_url, entry_id, read_at FROM read_history WHERE feed_url IN ({placeholders}) ORDER BY read_at DESC LIMIT ?",
                feed_url_values + (max(1, int(limit)),),
            ).fetchall()
            if hist_rows:
                for row in hist_rows:
                    history_fast_keys.append((row["feed_url"], row["entry_id"]))
                    raw_read_at = row["read_at"]
                    if raw_read_at:
                        try:
                            read_state_map[(row["feed_url"], row["entry_id"])] = datetime.fromisoformat(str(raw_read_at))
                        except Exception:
                            pass
            else:
                # Fall back to legacy entry_read_state scan (pre-read_history entries)
                read_rows = conn.execute(
                    f"SELECT feed_url, entry_id, read_at FROM entry_read_state WHERE feed_url IN ({placeholders})",
                    feed_url_values,
                ).fetchall()
                for row in read_rows:
                    raw_read_at = row["read_at"]
                    if not raw_read_at:
                        continue
                    try:
                        read_state_map[(row["feed_url"], row["entry_id"])] = datetime.fromisoformat(str(raw_read_at))
                    except Exception:
                        continue

    # Maintained item size (Saved/Kept only — no display and no sort key uses
    # it anywhere else, so no reason to pay a third DB's worth of query on
    # every ordinary feed-list render). NULL/missing means "not archived yet",
    # not "zero bytes" — see ensure_starred_archive_schema.
    size_map: dict[tuple[str, str], int] = {}
    if normalized_star_only:
        try:
            with archive_conn() as _ac:
                size_rows = _ac.execute(
                    f"SELECT feed_url, entry_id, content_size_bytes FROM archived_entry"
                    f" WHERE feed_url IN ({placeholders}) AND content_size_bytes IS NOT NULL",
                    feed_url_values,
                ).fetchall()
            size_map = {(row["feed_url"], row["entry_id"]): int(row["content_size_bytes"]) for row in size_rows}
        except Exception as exc:  # noqa: BLE001 — the view still renders without sizes
            LOGGER.warning("archive size lookup failed: %s", exc)

    # Tag-as-keep: the Saved view is now a unified "Kept" view showing entries
    # that are starred OR manually tagged. Load the tagged keys for the view's
    # feeds so the star fast path and filter can treat tags as a keep signal too.
    tagged_entries_set = get_tagged_entry_keys(set(feed_urls)) if normalized_star_only else set()
    # Keys kept by either axis (star or tag) — drives the Kept view's fast path
    # and membership filter. `saved_entries_set` stays star-only (the `saved` flag).
    #
    # *kept_scope* narrows that: "starred" restricts to the star axis alone, for
    # Read Mode's Inbox. The Inbox is a to-do pile, and a tag is filing rather
    # than a to-do — without this the Inbox lists the whole 24k library. The
    # main app's Kept view keeps the default "kept" (star OR tag).
    kept_entries_set = set(saved_entries_set) if kept_scope == "starred" else saved_entries_set | tagged_entries_set
    # The Archive (done) axis, applied here rather than by the caller because it
    # has to happen BEFORE the limit clip. Filtering afterwards means the caller
    # only ever sees archived items that sorted into the first N of the whole
    # kept backlog — with 24k kept and a 150 limit, the Archive view showed
    # whatever happened to land at the top and nothing else. It looked fine under
    # newest-first (recent archives sort high) and returned nothing under
    # "Recently starred", where an archived item has no star date at all.
    archive_filter_keys = get_archived_saved_keys() if archived is not None else set()
    capture_failed_keys = (
        starred_archive_service.get_capture_failed_keys() if (normalized_read_filter == "capture_failed" and normalized_star_only) else None
    )
    # Archived counts as kept, the same third keep signal `entry_has_keep_signal`
    # and `_prune_entries` honor. Archiving REMOVES the star, so an archived item
    # with no tag is otherwise not "kept" by any axis and would be unreachable in
    # the very view built to show it.
    if archived and kept_scope != "starred":
        kept_entries_set = kept_entries_set | archive_filter_keys

    with get_reader() as reader:
        # Build a map of feed_url → site homepage URL so favicons use the
        # actual site domain rather than the feed host (e.g. hnrss.org).
        # Single get_feeds() call instead of one get_feed() per feed.
        fetch_start = time.perf_counter()
        # Build feed_url → site homepage URL map via direct SQL rather than
        # iterating all feed objects via reader. Keeps this O(feeds in view)
        # instead of O(all feeds in the library). Goes through get_reader()'s
        # pooled, timed connection (10s busy_timeout) rather than a fresh
        # sqlite3.connect(..., timeout=5.0) per request -- the same anti-pattern
        # get_tagged_entry_keys was fixed for 2026-09-02 (Plan.md Tier 1), just
        # never caught here: a raw connection pays file-open/schema-load cost on
        # every render of a folder with many feeds, with a shorter busy_timeout
        # than everywhere else, and was invisible to slow-SQL logging since a
        # bare sqlite3.connect() bypasses _TimedConnection entirely. Row access
        # is positional (not sqlite3.Row) -- reader's own pooled connection
        # doesn't set a row_factory, same as get_tagged_entry_keys assumes.
        feed_site_map: dict[str, str | None] = {url: None for url in feed_urls}
        if feed_urls:
            _sm_db = reader._storage.get_db()
            _sm_list = list(feed_urls)
            for _i in range(0, len(_sm_list), 999):
                _chunk = _sm_list[_i : _i + 999]
                _ph = ",".join("?" for _ in _chunk)
                for _row in _sm_db.execute(f"SELECT url, link FROM feeds WHERE url IN ({_ph})", _chunk).fetchall():
                    feed_site_map[str(_row[0])] = _row[1] or None

        all_feed_entries: list[Any] = []
        fetch_limit = max(1, int(limit))
        need_all = bool(search_terms or site_hosts or normalized_sort_dir == "asc")
        # When a manual tag is selected, push the filter into reader's native
        # tags= argument so the match happens in SQL across the whole library.
        # Previously the tag was applied only as a post-filter on the newest-N
        # window fetched below, so tagged entries outside that window (tags are
        # sparse) never surfaced — clicking a tag showed nothing.
        tag_filter = [f"{MANUAL_TAG_KEY_PREFIX}{normalized_selected_tag}"] if normalized_selected_tag else None
        PER_FEED_QUERY_THRESHOLD = 32

        # enrich=False callers (Read Above/Below, "move visible to feed") only
        # ever read feed_url/id/title/link/dates/read off the fetched entries —
        # see the light-record loop below. For that shape, skip reader's own
        # per-row hydration (JSON-decoding content/enclosures/author, a `feeds`
        # join) entirely and build light_records' rows straight from SQL.
        # Scoped to exactly the cases the light-record loop doesn't need
        # search/tag/site/star/archive/history handling for — those still take
        # the slower paths below, unchanged. See _light_entries_from_sql.
        light_fast: list[_LightEntry] | None = None
        if (
            not enrich
            and not history_fast_keys
            and not normalized_star_only
            and not search_terms
            and not site_hosts
            and not tag_filter
            and archived is None
            and normalized_sort_by in ("post", "received")
            and normalized_read_filter in ("all", "unread", "starred")
        ):
            light_fast = _light_entries_from_sql(
                feed_urls,
                reader_read_filter=reader_read_filter,
                sort_by=normalized_sort_by,
                sort_dir=normalized_sort_dir,
                limit=fetch_limit,
            )

        if light_fast is not None:
            all_feed_entries = light_fast
        elif history_fast_keys:
            # Fast history path: fetch each entry by primary key (indexed lookup)
            # instead of scanning all read entries. N small lookups vs. one huge scan.
            for furl, eid in history_fast_keys:
                e = reader.get_entry((furl, eid), None)
                if e is not None:
                    all_feed_entries.append(e)
        elif normalized_star_only:
            # Kept fast path (same reasoning as the tag fix above): kept
            # entries are sparse and often OLD — imported stars/tags sit far past
            # any newest-N fetch window, so windowed-scan-then-post-filter
            # shows nothing. kept_entries_set (star OR tag) is already restricted
            # to the view's feeds; point-lookup exactly those keys instead.
            star_keys: Iterable[tuple[str, str]] = kept_entries_set
            if normalized_selected_tag:
                # Narrow to the tag in SQL BEFORE hydrating. Without this the tag
                # filter ran in Python after hydrating (and lead-image-fetching)
                # the whole kept set — ~38s to render an empty `inbox` view.
                star_keys = kept_entries_set & get_entry_keys_for_manual_tag(set(feed_urls), normalized_selected_tag)
            if archived is not None:
                # Narrow to the done-axis BEFORE the window clip below, for the
                # same reason the tag filter moved up here: _sorted_star_key_window
                # sorts and clips in SQL over the raw keys, so anything filtered
                # afterwards is chosen from a window computed against the whole
                # kept set. With one archived post among 24k kept, "oldest first"
                # put it far outside the first 150 and the Archive node rendered
                # "Nothing here" while its count said 1.
                star_keys = {k for k in star_keys if (k in archive_filter_keys) == archived}
            if capture_failed_keys is not None:
                # Same reason as the archived narrowing above: filter before the window clip, or failures outside the
                # first window never show.
                star_keys = {k for k in star_keys if k in capture_failed_keys}
            if search_terms:
                # Narrow in SQL BEFORE hydrating. This branch runs ahead of the
                # `elif search_terms` fast path below, so the Saved view was the
                # one place a search never took a fast path at all: it hydrated
                # every kept key and filtered in Python — ~11k get_entry calls,
                # measured at 28s per search on a real library, which reads as a
                # search box that does nothing.
                matched_keys = _filter_star_keys_by_search(set(star_keys), search_terms)
                if matched_keys is not None:
                    star_keys = matched_keys
                    search_terms = []  # already matched in SQL
            if len(star_keys) > fetch_limit and not search_terms:
                # Big backlog: hydrating every kept key costs seconds (6k
                # get_entry calls ≈ 4s). Sort + read-filter + clip in SQL over
                # the raw keys and hydrate only the visible window. The tag
                # narrowing above already reduced the set, so this now applies to
                # tag views too.
                star_keys = _sorted_star_key_window(
                    star_keys,
                    sort_by=normalized_sort_by,
                    sort_dir=normalized_sort_dir,
                    reader_read_filter=reader_read_filter,
                    limit=fetch_limit,
                )
            for furl, eid in star_keys:
                e = reader.get_entry((furl, eid), None)
                if e is not None:
                    all_feed_entries.append(e)
        elif search_terms:
            # Search fast path: narrow to matching keys in SQL, then hydrate
            # only the survivors. Replaces reader's FTS index, whose
            # per-result highlighted snippet was ~95% of a ~10-20s search —
            # see _search_entry_keys_in_sql. Falls back to the Python scan.
            matched_keys = _search_entry_keys_in_sql(search_terms, feed_urls, reader_read_filter, fetch_limit)
            if matched_keys is not None:
                for furl, eid in matched_keys:
                    if furl not in feed_urls:
                        continue  # unscoped query (>900 feeds): drop out-of-scope
                    e = reader.get_entry((furl, eid), None)
                    if e is not None:
                        all_feed_entries.append(e)
                search_terms = []  # already matched in SQL
            else:
                for entry in reader.get_entries(read=reader_read_filter):
                    if entry.feed_url not in feed_urls:
                        continue
                    all_feed_entries.append(entry)
        elif normalized_sort_dir == "asc" and not search_terms and len(feed_urls) > PER_FEED_QUERY_THRESHOLD and not tag_filter:
            # ASC (oldest-first) with many feeds: reader only supports newest-first,
            # so normally we'd pull everything into Python and sort. Instead, use a
            # direct SQL query sorted ASC and fetch Entry objects only for matched rows.
            # When feed count fits within SQLite's 999-variable limit use an IN clause
            # so only the target folder's entries are scanned (avoids returning the
            # globally-oldest rows from unrelated feeds). For >999 feeds fall back to
            # a buffer-based global scan.
            read_sql = {None: "", True: " AND read = 1", False: " AND (read IS NULL OR read != 1)"}
            read_clause = read_sql.get(reader_read_filter, "")
            # Pooled connection, not a fresh sqlite3.connect() per request -- see
            # the feed_site_map comment above for the full rationale. Row access
            # is positional for the same reason.
            try:
                _rdb = reader._storage.get_db()
                if len(feed_urls) <= 999:
                    _feed_list = list(feed_urls)
                    _placeholders = ",".join("?" for _ in _feed_list)
                    rows = _rdb.execute(
                        f"SELECT feed, id FROM entries WHERE feed IN ({_placeholders}){read_clause} ORDER BY {_ENTRY_SORT_SQL} ASC LIMIT ?",
                        _feed_list + [fetch_limit],
                    ).fetchall()
                else:
                    # >999 feeds: SQLite's variable limit prevents a single IN clause.
                    # A global scan without a feed filter picks up entries from feeds
                    # outside feed_urls (e.g. import-synthesised feeds) and the Python
                    # filter can discard the entire result window. Batch instead.
                    _feed_list = list(feed_urls)
                    batch_rows: list = []
                    for _i in range(0, len(_feed_list), 999):
                        _chunk = _feed_list[_i : _i + 999]
                        _ph = ",".join("?" for _ in _chunk)
                        chunk_rows = _rdb.execute(
                            f"SELECT feed, id, {_ENTRY_SORT_SQL} AS sort_val FROM entries"
                            f" WHERE feed IN ({_ph}){read_clause}"
                            f" ORDER BY {_ENTRY_SORT_SQL} ASC LIMIT ?",
                            _chunk + [fetch_limit],
                        ).fetchall()
                        batch_rows.extend(chunk_rows)
                    batch_rows.sort(key=lambda r: r[2] or "")
                    rows = batch_rows
                for row in rows:
                    if str(row[0]) not in feed_urls:
                        continue
                    e = reader.get_entry((str(row[0]), str(row[1])), None)
                    if e is not None:
                        all_feed_entries.append(e)
                    if len(all_feed_entries) >= fetch_limit:
                        break
            except Exception:
                LOGGER.exception("[perf] asc-sql fast path failed, falling back")
                for entry in reader.get_entries(read=reader_read_filter):
                    if entry.feed_url not in feed_urls:
                        continue
                    all_feed_entries.append(entry)
        elif not search_terms and len(feed_urls) > PER_FEED_QUERY_THRESHOLD and not tag_filter:
            # DESC (newest-first) with many feeds: same SQL-batch approach as the
            # ASC path. Avoids a Python-side global scan of every entry in the DB.
            # "post" sort uses the entry publish date; "received" uses the time
            # reader first observed the entry (first_updated), matching the
            # light-record loop's Python sort key for "received".
            read_sql = {None: "", True: " AND read = 1", False: " AND (read IS NULL OR read != 1)"}
            read_clause = read_sql.get(reader_read_filter, "")
            sort_col = "first_updated" if normalized_sort_by == "received" else _ENTRY_SORT_SQL
            # Pooled connection, positional row access -- see the feed_site_map
            # comment above for the full rationale.
            try:
                _rdb = reader._storage.get_db()
                _feed_list = list(feed_urls)
                if len(_feed_list) <= 999:
                    _placeholders = ",".join("?" for _ in _feed_list)
                    rows = _rdb.execute(
                        f"SELECT feed, id FROM entries WHERE feed IN ({_placeholders}){read_clause} ORDER BY {sort_col} DESC LIMIT ?",
                        _feed_list + [fetch_limit],
                    ).fetchall()
                else:
                    batch_rows_desc: list = []
                    for _i in range(0, len(_feed_list), 999):
                        _chunk = _feed_list[_i : _i + 999]
                        _ph = ",".join("?" for _ in _chunk)
                        chunk_rows = _rdb.execute(
                            f"SELECT feed, id, {sort_col} AS sort_val FROM entries"
                            f" WHERE feed IN ({_ph}){read_clause}"
                            f" ORDER BY {sort_col} DESC LIMIT ?",
                            _chunk + [fetch_limit],
                        ).fetchall()
                        batch_rows_desc.extend(chunk_rows)
                    batch_rows_desc.sort(key=lambda r: r[2] or "", reverse=True)
                    rows = batch_rows_desc[:fetch_limit]
                for row in rows:
                    e = reader.get_entry((str(row[0]), str(row[1])), None)
                    if e is not None:
                        all_feed_entries.append(e)
                    if len(all_feed_entries) >= fetch_limit:
                        break
            except Exception:
                LOGGER.exception("[perf] desc-sql fast path failed, falling back")
                for entry in reader.get_entries(read=reader_read_filter, tags=tag_filter):
                    if entry.feed_url not in feed_urls:
                        continue
                    all_feed_entries.append(entry)
                    if not need_all and len(all_feed_entries) >= fetch_limit:
                        break
        else:
            # Few feeds: query per feed with the SQL feed= filter.
            for feed_url in feed_urls:
                for entry in reader.get_entries(feed=feed_url, read=reader_read_filter, tags=tag_filter):
                    all_feed_entries.append(entry)
                    if not need_all and len(all_feed_entries) >= fetch_limit:
                        break
                if not need_all and len(all_feed_entries) >= fetch_limit:
                    break

        fetch_ms = int((time.perf_counter() - fetch_start) * 1000)
        LOGGER.info(
            "[perf] list_entries: feeds=%d entries_fetched=%d fetch_ms=%d",
            len(feed_urls),
            len(all_feed_entries),
            fetch_ms,
        )
        if _stats is not None:
            # Fewer raw rows than the window means the source is exhausted: a bigger window cannot add any.
            _stats["exhausted"] = len(all_feed_entries) < fetch_limit

        # Two-phase processing: build a lightweight per-entry record that has
        # *only* what filter/sort/dedupe needs, then enrich the surviving top-N
        # entries with display fields (thumbnails, formatted dates, favicons,
        # YouTube duration). Avoids paying ~24ms × 2000 entries of display work
        # for entries that won't be displayed.
        process_start = time.perf_counter()
        if normalized_read_filter == "history" and not normalized_star_only:
            sort_key = "history_sort_value"
            sort_desc = True
        elif normalized_sort_by == "starred":
            sort_key = "saved_sort_value"
            sort_desc = normalized_sort_dir == "desc"
        elif normalized_sort_by == "size":
            sort_key = "size_sort_value"
            sort_desc = normalized_sort_dir == "desc"
        else:
            sort_key = "post_sort_value" if normalized_sort_by == "post" else "received_sort_value"
            sort_desc = normalized_sort_dir == "desc"

        # Loaded here (before the filter loop, not just the later enrichment
        # phase) so "don't show unpremiered videos yet" can drop entries before
        # the sort+limit clip — a filtered-out row must not steal one of the
        # top-N display slots from a real (already-aired) entry.
        _pre_start = time.perf_counter()
        with get_meta_connection() as _prefs_conn:
            _all_display_prefs = get_all_feed_display_prefs(_prefs_conn)
        _prefs_ms = int((time.perf_counter() - _pre_start) * 1000)
        _hide_unpremiered_global = youtube_hide_unpremiered_global()
        _hide_locked_comics_global = hide_locked_comics_global()

        # Batch-loaded once, same rationale as _all_display_prefs above — a
        # per-entry query here would be an N+1 over every webcomic entry in
        # the view. Needed whenever the row will be enriched for display (the
        # locked-placeholder badge is shown regardless of hide_locked_comics —
        # that preference only controls whether the entry appears at ALL), or
        # when at least one feed in scope could actually be gated: the global
        # toggle, or a webcomic feed with the per-feed pref on.
        _locked_until_map: dict[tuple[str, str], float] = {}
        if (
            enrich
            or _hide_locked_comics_global
            or any(_all_display_prefs.get(fu, _DISPLAY_PREF_DEFAULTS).get("hide_locked_comics") for fu in feed_urls)
        ):
            try:
                with get_meta_connection() as _lock_conn:
                    # Locked rows are a handful library-wide, so ask for them directly and scope in Python. Probing
                    # entry_lead_images per feed (150k rows) cost ~100ms per 999-feed chunk -- ~300ms for "All" -- to
                    # find the one locked row, and the chunking is no longer needed to stay under the bind limit.
                    for _row in _lock_conn.execute(
                        "SELECT feed_url, entry_id, locked_until FROM entry_lead_images WHERE locked_until IS NOT NULL"
                    ).fetchall():
                        if str(_row[0]) in feed_urls:
                            _locked_until_map[(str(_row[0]), str(_row[1]))] = float(_row[2])
            except Exception:
                LOGGER.exception("[hide-locked-comics] failed to load locked_until map")

        # A feed-tag preview filter (clicking a suggested-tag chip in the entry
        # pane): unlike normalized_selected_tag above, this is never pushed into
        # reader's native fetch — it's a one-off narrowing of whatever the
        # current view already fetched, not a persisted scope, so a single
        # pre-fetched set + O(1) membership check is enough (no per-entry call).
        _locked_ms = int((time.perf_counter() - _pre_start) * 1000) - _prefs_ms
        if _prefs_ms + _locked_ms > 100:
            LOGGER.info("[perf] list_entries: prep prefs_ms=%d locked_ms=%d", _prefs_ms, _locked_ms)
        feed_tag_keys: set[tuple[str, str]] | None = None
        if normalized_selected_feed_tag:
            feed_tag_keys = get_entry_ids_with_feed_tag(set(feed_urls), normalized_selected_feed_tag)

        light_records: list[dict] = []
        for entry in all_feed_entries:
            is_read = bool(entry.read)
            is_saved = (entry.feed_url, entry.id) in saved_entries_set
            manual_tags_for_record: list[str] = []
            if normalized_selected_tag:
                # Only reached when tag_filter is set, which excludes this
                # entry ever being a _LightEntry shim — see the fast-path gate.
                manual_tags_for_record = get_manual_tags_for_resource(reader, entry.resource_id)  # ty: ignore[unresolved-attribute]
                if normalized_selected_tag not in manual_tags_for_record:
                    continue
            if feed_tag_keys is not None and (entry.feed_url, entry.id) not in feed_tag_keys:
                continue
            # Kept view keeps anything starred OR tagged (the unified keep axis).
            if normalized_star_only and (entry.feed_url, entry.id) not in kept_entries_set:
                continue
            # Feeds-mode "Starred" filter: literal stars only (not tag-kept),
            # within the current folder/feed scope, ignoring read state
            # entirely — independent of star_only/kept_scope (the Saved-view
            # mode switch) so it never touches that scope's remembered sort.
            if normalized_read_filter == "starred" and not is_saved:
                continue
            if archived is not None and (((entry.feed_url, entry.id) in archive_filter_keys) != archived):
                continue
            # Saved-view "Capture failed": kept items whose source-page capture failed (see get_capture_failed_keys).
            if capture_failed_keys is not None and (entry.feed_url, entry.id) not in capture_failed_keys:
                continue
            # Unread composes with star_only (Saved view narrowed to unread);
            # history stays exclusive with starred (it sorts by read time).
            if normalized_read_filter == "unread" and is_read:
                continue
            if not normalized_star_only and normalized_read_filter == "history" and not is_read:
                continue
            # A starred entry is exempt from hide_unpremiered, same as it's
            # already exempt from retention/purge and bulk mark-read — starring
            # is deliberate "track this," and the Starred filter must show it
            # regardless of anything else, or starring it would defeat the
            # point of being able to find it again.
            if normalized_read_filter != "starred" and "youtube.com/feeds/videos.xml" in str(entry.feed_url):
                _feed_hide_unpremiered = _hide_unpremiered_global or bool(
                    _all_display_prefs.get(entry.feed_url, _DISPLAY_PREF_DEFAULTS).get("hide_unpremiered")
                )
                if _feed_hide_unpremiered and _is_youtube_unpremiered(entry):
                    continue
            # Same exemption and shape as hide_unpremiered above, for a
            # webcomic strip a publisher has locked behind a supporter tier
            # (cad-comic.com and the like) until a stated date.
            if normalized_read_filter != "starred" and _locked_until_map:
                _feed_hide_locked = _hide_locked_comics_global or bool(
                    _all_display_prefs.get(entry.feed_url, _DISPLAY_PREF_DEFAULTS).get("hide_locked_comics")
                )
                if _feed_hide_locked:
                    _locked_until = _locked_until_map.get((str(entry.feed_url), str(entry.id)))
                    if _locked_until is not None and _locked_until > time.time():
                        continue
            published_dt = entry_effective_date(entry)
            read_dt = read_state_map.get((entry.feed_url, entry.id))
            if read_dt is None:
                read_dt = getattr(entry, "read_modified", None)

            if site_hosts:
                host = _entry_link_site_host(entry)
                if not any(host == s or host.endswith("." + s) for s in site_hosts):
                    continue
            title_text = _display_title(entry) or entry.title
            if search_terms:
                # Only reached when search_terms is set, which excludes this
                # entry ever being a _LightEntry shim — see the fast-path gate.
                search_haystack = " ".join(
                    [
                        str(title_text or ""),
                        str(entry.feed_resolved_title or entry.feed_url or ""),  # ty: ignore[unresolved-attribute]
                        str(entry.link or ""),
                        str(getattr(entry, "authors_str", None) or ""),
                        str(entry.summary or ""),  # ty: ignore[unresolved-attribute]
                    ]
                ).lower()
                if not all(term in search_haystack for term in search_terms):
                    continue

            sort_value: float
            if sort_key == "saved_sort_value":
                # Unstarred-but-tagged entries have no saved_at. They can only
                # appear here outside the Inbox (which is starred-only), and
                # datetime_sort_value(None) parks them at the far end rather
                # than interleaving them arbitrarily.
                sort_value = datetime_sort_value(saved_at_map.get((entry.feed_url, entry.id)))
            elif sort_key == "history_sort_value":
                sort_value = datetime_sort_value(read_dt)
            elif sort_key == "post_sort_value":
                # published_dt is already entry_effective_date, which now runs
                # the whole inference chain and only then falls back to received
                # time. The chain used to be repeated here instead — and was
                # unreachable, because entry_effective_date returned `added` and
                # every `or` below it was dead. See entry_publication_date.
                sort_value = datetime_sort_value(published_dt)
            elif sort_key == "size_sort_value":
                # No completed archive yet is 0, not "unknown" -- there is
                # nothing captured for it to weigh, which is literally true.
                sort_value = float(size_map.get((entry.feed_url, entry.id), 0))
            else:
                sort_value = datetime_sort_value(entry.added)

            light_records.append(
                {
                    "_entry": entry,
                    "_published_dt": published_dt,
                    "_read_dt": read_dt,
                    "_manual_tags": manual_tags_for_record,
                    "feed_url": entry.feed_url,
                    "id": entry.id,
                    "title": title_text,
                    # Sanitized here (not after the rebase below) so an unsafe
                    # link is emptied before anything else consumes it — the
                    # rebase then no-ops on the falsy value. See safe_link_url.
                    "link": html_sanitize.safe_link_url(entry.link or _derived_entry_link(entry)),
                    "read": is_read,
                    "saved": is_saved,
                    # A Lectio capture rather than a feed-provided entry. Drives
                    # the Re-fetch control, which must follow the *entry*, not
                    # the feed it happens to sit on — auto-filing moves captures
                    # onto real feeds and used to strip their re-fetch hatch.
                    "captured": str(getattr(entry, "added_by", "") or "") == "user",
                    sort_key: sort_value,
                }
            )
            if normalized_star_only:
                # None (not 0) when nothing is archived yet, so the template
                # can render "—" instead of a misleading "0 B".
                light_records[-1]["size_bytes"] = size_map.get((entry.feed_url, entry.id))

    filter_ms = int((time.perf_counter() - process_start) * 1000)
    # Dedupe + sort + limit on the lightweight records.
    best_by_key: dict[str, dict] = {}
    passthrough: list[dict] = []
    for rec in light_records:
        dedupe_key = build_entry_dedupe_key(cast(str | None, rec.get("link")), cast(str | None, rec.get("title")))
        if not dedupe_key:
            passthrough.append(rec)
            continue
        existing = best_by_key.get(dedupe_key)
        if existing is None or rec[sort_key] > existing[sort_key]:
            best_by_key[dedupe_key] = rec
    light_records = passthrough + list(best_by_key.values())
    light_records.sort(key=lambda item: item[sort_key], reverse=sort_desc)
    light_records = light_records[:limit]

    if not enrich:
        # Callers that only need feed_url/id/read/link — resolving an anchor's
        # position for "Read above/below", say — pay for thumbnails, tags,
        # feed display prefs and premiere-prefix computation on every entry in
        # the whole scope otherwise. Measured on the live library: 8,472-entry
        # "All Feeds" unread view, phase 2 alone was ~3.9s of a ~10.4s total
        # that this caller never uses any of.
        no_enrich_ms = int((time.perf_counter() - process_start) * 1000)
        LOGGER.info(
            "[perf] list_entries: entries_processed=%d process_ms=%d (enrich skipped)",
            len(light_records),
            no_enrich_ms,
        )
        return [{k: v for k, v in rec.items() if not k.startswith("_") and k != sort_key} for rec in light_records]

    enrich_start = time.perf_counter()
    # Which of the VISIBLE rows carry a manual tag: one query over the clipped
    # window, not the whole backlog. `_manual_tags` above is filled only when a tag
    # filter is active, so a kept flag derived from it read empty on ordinary rows
    # — which is why re-fetch appeared in the entry pane but not from the post
    # list, and only once a post had been starred.
    _visible_manual: dict[tuple[str, str], list[str]] = {}
    if light_records:
        try:
            _keys = [(str(r["feed_url"]), str(r["id"])) for r in light_records]
            # reader's own connection, not a second one: an independent handle can
            # lose a lock race with the refresh worker, and this code's except
            # branch would then quietly report "no tags" — hiding the re-fetch
            # menu at random. Surfaced as an intermittent test failure.
            with get_reader() as _r:
                _tdb = _r._storage.get_db()
                for _i in range(0, len(_keys), 400):
                    _chunk = _keys[_i : _i + 400]
                    _ph = ",".join("(?,?)" for _ in _chunk)
                    _flat = [v for k in _chunk for v in k]
                    for _f, _e, _k in _tdb.execute(
                        f"SELECT DISTINCT feed, id, key FROM entry_tags"  # nosemgrep: placeholders only
                        f" WHERE (feed, id) IN ({_ph}) AND key LIKE ?",
                        _flat + [f"{MANUAL_TAG_KEY_PREFIX}%"],
                    ):
                        _visible_manual.setdefault((str(_f), str(_e)), []).append(str(_k)[len(MANUAL_TAG_KEY_PREFIX) :].strip().lower())
        except Exception:  # noqa: BLE001 — a missing flag only hides a menu item
            LOGGER.warning("visible-tag probe failed", exc_info=True)
    _visible_tagged: set[tuple[str, str]] = set(_visible_manual)
    # Chips (publisher tags as shown, not pinned-only ones) for "Filter this view"'s tag: syntax; applied tags feed tagged:.
    _visible_chips = _chip_tags_for_rows([(str(r["feed_url"]), str(r["id"])) for r in light_records]) if light_records else {}

    # _all_display_prefs was already loaded above, before the filter loop.
    # Which feeds have a pinned thumbnail copy, read once. This used to be a has_pinned_feed_thumbnail()
    # call inside the per-entry loop — one image-cache query per rendered row.
    _pinned_thumb_keys = pinned_feed_thumbnail_keys()
    # Declared host migrations (feed_url_rewrites), so the proxy-rebase below
    # can't send correct entry links back to an author's dead domain still named
    # in a feed's channel <link>. Loaded once — {} for the common no-rules case.
    _host_aliases = get_dedupe_host_aliases()
    # Enrich the surviving (top-N) records with display fields.
    entries = []
    for rec in light_records:
        entry = cast(object, rec.pop("_entry"))
        published_dt = rec.pop("_published_dt")
        read_dt = rec.pop("_read_dt")
        rec.pop(sort_key, None)

        feed_url_str = cast(str, rec["feed_url"])
        title_text = cast(str, rec["title"])

        duration_seconds = None
        duration_display = None
        video_id = None
        try:
            entry_feed_url = getattr(entry, "feed_url", None)
            entry_link = getattr(entry, "link", None)
            if isinstance(entry_feed_url, str) and "youtube.com/feeds/videos.xml" in entry_feed_url and entry_link:
                vid = youtube_duration_service.extract_video_id(entry_link)
                if vid:
                    video_id = vid
                    premiere_prefix = _youtube_premiere_prefix(vid)
                    if premiere_prefix:
                        title_text = f"[{premiere_prefix}] {title_text}"
                        rec["title"] = title_text
                    else:
                        duration_seconds, duration_display = youtube_duration_service.get_cached_duration(vid)
                        if duration_display:
                            title_text = f"[{duration_display}] {title_text}"
                            rec["title"] = title_text
        except Exception:
            duration_seconds = None
            duration_display = None
            video_id = None

        manual_tags = cast(list[str], rec.pop("_manual_tags"))

        # Rebase proxy-feed entry links (e.g. feedburner) to the real publisher host.
        if rec.get("link") and hasattr(entry, "feed"):
            _ch = _rewrite_url_host(getattr(entry.feed, "link", None), _host_aliases)
            rec["link"] = _rebase_proxy_entry_link(str(rec["link"]), feed_url_str, _ch)

        _feed_prefs = _all_display_prefs.get(feed_url_str, _DISPLAY_PREF_DEFAULTS)
        _feed_thumb_setting = _feed_prefs.get("feed_thumbnail_url")
        _show_thumb = bool(_feed_prefs.get("show_lead_image_as_thumb", 1))
        _thumb_strategy = _feed_prefs.get("thumb_strategy") or None
        if _feed_thumb_setting and _feed_thumb_setting != "__favicon__":
            # Prefer the pinned copy: the chosen URL may be a signed CDN link that has since expired.
            _raw_thumb = (
                f"/api/feed-thumb?feed_url={quote_plus(feed_url_str)}"
                if _feed_thumb_cache_key(feed_url_str) in _pinned_thumb_keys
                else str(_feed_thumb_setting)
            )
        elif _feed_thumb_setting == "__favicon__":
            _raw_thumb = None
        else:
            if _thumb_strategy == "inline":
                # Use first image from feed content HTML — bypasses the og_scrape cache
                _raw_thumb = lead_image_service.extract_inline_thumb_url(entry)
            elif _thumb_strategy == "media_rss":
                _raw_thumb = lead_image_service.extract_media_rss_thumb_url(entry)
            else:
                _raw_thumb = None
            if not _raw_thumb:
                # Strategy-specific extraction found nothing — e.g. a feed whose
                # thumb_strategy was auto-detected as media_rss/inline but whose
                # reader Entry carries no usable media field. Fall back to the
                # cached lead image (same one shown in the article view) instead
                # of rendering a blank thumbnail.
                _raw_thumb = lead_image_service.get_cached_entry_thumbnail(
                    feed_url_str,
                    str(getattr(entry, "id", "") or ""),
                    str(getattr(entry, "link", "") or ""),
                )
            if not _raw_thumb:
                # Last resort: a raw inline <svg> in the entry content (sanitized
                # → data URI). Keeps the list thumb consistent with the article.
                _raw_thumb = lead_image_service.extract_inline_svg_thumb_url(entry)
        _thumb = _raw_thumb if _show_thumb else None
        _feed_thumb_crop = str(_feed_prefs.get("thumb_crop") or "cover")
        if _feed_thumb_crop not in _VALID_THUMB_CROPS:
            _feed_thumb_crop = "cover"
        _entry_id = str(getattr(entry, "id", "") or "")
        # Prefer the pinned copy: a signed lead-image URL (DeviantArt wixmp,
        # etc.) may have a since-expired token. Only the pinned entries are
        # substituted — everything else is unchanged, so this is a no-op until
        # the enhance pass has actually pinned something for this entry.
        if _thumb and _url_is_signed(_thumb) and has_pinned_entry_thumbnail(feed_url_str, _entry_id, usable_only=True):
            _thumb = f"/api/entry-thumb?feed_url={quote_plus(feed_url_str)}&entry_id={quote_plus(_entry_id)}"
        _entry_crop_override = lead_image_service.get_entry_thumb_crop(feed_url_str, _entry_id)
        _thumb_crop = _entry_crop_override if _entry_crop_override else _feed_thumb_crop
        _smart_ms = _feed_prefs.get("smart_min_scale")
        _fill_zm = _feed_prefs.get("fill_zoom")
        # Surfaced on every entry (not just when hide_locked_comics is on) so a
        # reader who leaves that preference off still sees a locked-until badge
        # instead of a broken thumbnail for a currently-locked webcomic strip.
        _locked_until = _locked_until_map.get((feed_url_str, _entry_id))
        _is_locked = _locked_until is not None and _locked_until > time.time()
        rec.update(
            {
                "thumbnail_url": _thumb,
                "show_thumbnail": _show_thumb,
                "thumb_crop": _thumb_crop,
                "is_locked": _is_locked,
                "locked_until_ts": _locked_until if _is_locked else None,
                "locked_until_display": (
                    format_datetime_for_ui(datetime.fromtimestamp(_locked_until, tz=timezone.utc)) if _is_locked else None
                ),
                "smart_min_scale": float(_smart_ms) if _smart_ms is not None else None,
                "fill_zoom": float(_fill_zm) if _fill_zm is not None else None,
                "thumb_strategy": _thumb_strategy or "",
                "feed_title": getattr(entry, "feed_resolved_title", None) or feed_url_str,
                "feed_icon_url": get_favicon_url(feed_url_str, _rewrite_url_host(feed_site_map.get(feed_url_str), _host_aliases)),
                "manual_tags": manual_tags,
                # Whether this row itself carries any manual tag (distinct from
                # "kept" below, which also counts a star) — drives the postlist
                # tag icon's filled/outline state.
                "has_tags": (feed_url_str, _entry_id) in _visible_tagged,
                "chip_tags": _visible_chips.get((feed_url_str, _entry_id), []),
                "applied_tags": sorted(set(_visible_manual.get((feed_url_str, _entry_id), []))),
                # Starred OR tagged. Drives the re-fetch menu, which must not key
                # on manual_tags (populated only under a tag filter) nor on the
                # star alone — tag-as-keep made a tag a keep signal everywhere.
                "kept": bool(is_saved or (feed_url_str, _entry_id) in _visible_tagged),
                # Inline formatting a feed put in the title (<em>), rendered
                # rather than escaped. Everything else stays literal text, so a
                # C++ title's std::vector<T> survives — see sanitize_inline_title.
                "title_html": html_sanitize.sanitize_inline_title(title_text),
                "post_timestamp": published_dt.isoformat() if published_dt else None,
                "received_timestamp": _added_dt.isoformat() if (_added_dt := getattr(entry, "added", None)) else None,
                "read_timestamp": read_dt.isoformat() if read_dt else None,
                # When this was starred — carried so the orphan merge can re-sort
                # by it after list_entries_for_feeds pops the sort values.
                "saved_timestamp": (_sv.isoformat() if (_sv := saved_at_map.get((feed_url_str, _entry_id))) else None),
                "post_display": format_datetime_for_ui(published_dt),
                "received_display": format_datetime_for_ui(getattr(entry, "added", None)),
                "read_display": format_datetime_for_ui(read_dt),
                "size_display": (_format_size_bytes(_sb) if (_sb := rec.get("size_bytes")) is not None else None),
                "duration_seconds": duration_seconds,
                "duration_display": duration_display,
                "video_id": video_id,
            }
        )
        entries.append(rec)

    enrich_ms = int((time.perf_counter() - enrich_start) * 1000)
    process_ms = int((time.perf_counter() - process_start) * 1000)
    LOGGER.info(
        "[perf] list_entries: entries_processed=%d filter_ms=%d enrich_ms=%d process_ms=%d",
        len(entries),
        filter_ms,
        enrich_ms,
        process_ms,
    )

    return entries


def get_archived_saved_keys() -> set[tuple[str, str]]:
    """(feed_url, entry_id) of items the user has Archived — the read-later
    "done" axis. Archived items drop out of the Read Mode inbox and are reached
    via the Archive node or Search. Independent of the star: an entry can be
    archived with no saved_entries row at all."""
    with get_meta_connection() as conn:
        return {(str(row["feed_url"]), str(row["entry_id"])) for row in conn.execute("SELECT feed_url, entry_id FROM archived_entries")}


def set_entry_archived(feed_url: str, entry_id: str, archived: bool) -> None:
    """Set/clear the Archive (done) flag. Unlike the old column this works for
    any entry, starred or not — archiving is what *removes* the star, so
    requiring one would make the state unreachable the moment it was set."""
    with get_meta_connection() as conn:
        if archived:
            conn.execute(
                "INSERT OR REPLACE INTO archived_entries (feed_url, entry_id, archived_at) VALUES (?, ?, ?)",
                (feed_url, entry_id, datetime.now(timezone.utc).isoformat()),
            )
        else:
            conn.execute(
                "DELETE FROM archived_entries WHERE feed_url = ? AND entry_id = ?",
                (feed_url, entry_id),
            )
        conn.commit()


def is_entry_archived(feed_url: str, entry_id: str) -> bool:
    """Single-entry Archive check, for the keep-signal test on the unstar path."""
    with get_meta_connection() as conn:
        return (
            conn.execute(
                "SELECT 1 FROM archived_entries WHERE feed_url = ? AND entry_id = ?",
                (feed_url, entry_id),
            ).fetchone()
            is not None
        )


def entry_has_keep_signal(feed_url: str, entry_id: str, *, starred: bool) -> bool:
    """Is *anything* still keeping this entry?

    Three independent signals, and the offline capture plus pruning-exemption
    are owed to the entry if **any** of them holds:

    - **starred** — it is still a TODO;
    - **manually tagged** — it is filed;
    - **archived** — it is done, and "Archive keeps its contents" is the rule.

    Archive is the one that is easy to forget, and forgetting it is destructive
    rather than untidy: archiving unstars, so an archived-and-untagged entry
    would otherwise read as "no keep signal" and lose the very contents Archive
    promised to keep.
    """
    if starred:
        return True
    if get_manual_tags_for_entry(feed_url, entry_id):
        return True
    return is_entry_archived(feed_url, entry_id)


def apply_star_state(feed_url: str, entry_id: str, saved: bool) -> None:
    """Set/clear the star (the TODO axis) and everything that mirrors it.

    Extracted from ``POST /entries/saved`` so Archive can reuse it: archiving is
    *implemented* as an unstar, and re-implementing the capture/husk handling
    beside it is how the two drift apart.

    Unstarring releases the offline capture when no keep signal remains — see
    ``entry_has_keep_signal``. Callers that are about to archive must
    therefore write the archived row *first*, or this will tear down the
    contents Archive means to keep. A resulting Saved Articles husk (no star,
    no tag) is NOT hard-deleted here — see ``_sweep_husked_saved_articles``,
    which does that later so an in-window undo isn't lied to.
    """
    newly_starred = False
    with get_meta_connection() as conn:
        if saved:
            cur = conn.execute(
                "INSERT OR IGNORE INTO saved_entries (feed_url, entry_id) VALUES (?, ?)",
                (feed_url, entry_id),
            )
            newly_starred = cur.rowcount > 0
        else:
            conn.execute(
                "DELETE FROM saved_entries WHERE feed_url = ? AND entry_id = ?",
                (feed_url, entry_id),
            )
        conn.commit()
    with _has_manual_tags_lock:
        _has_manual_tags_cache.pop("saved_unread", None)

    # "On star, also send to…" — fire configured destinations once, only on a
    # genuine new star (rowcount), off-request so the star stays snappy.
    if newly_starred:
        _uid = tenancy.current_user_id()
        threading.Thread(
            target=lambda: _run_in_user_context(_uid, _run_on_star_destinations, feed_url, entry_id),
            daemon=True,
        ).start()

    # Mirror the save state into the archive: queue a capture when starred,
    # mark for later removal when unstarred. The archive worker handles the
    # actual fetching off-request.
    try:
        if saved:
            starred_archive_service.enqueue_archive(feed_url, entry_id)
        elif not entry_has_keep_signal(feed_url, entry_id, starred=False):
            # Only release the capture when no keep signal remains — a tag or an
            # Archive both keep it after the star is gone. Archive matters most
            # here: archiving is *implemented* as an unstar, so missing it would
            # delete the contents Archive exists to preserve.
            starred_archive_service.enqueue_removal(feed_url, entry_id)
            # A Saved Article that is now neither starred nor tagged is a husk —
            # the read-later pile should hold only kept items, and it's
            # user-added, so it CAN be removed outright rather than left as
            # invisible cruft. But not from here, and not immediately: this
            # function has no idea whether its caller just wrote an undo token
            # for this unstar (the single-post toggle and the bulk star/unstar
            # action both do), and hard-deleting out from under an in-window
            # undo would make the undo toast lie — the star could be restored
            # but the entry it pointed to would already be gone. The nightly
            # maintenance sweep does the actual deletion, after confirming no
            # entry_unstar_batch row for it is still inside the undo window.
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("starred archive enqueue failed for %s/%s: %s", feed_url, entry_id, exc)


def _sweep_husked_saved_articles() -> int:
    """Hard-delete lectio:saved entries left behind by apply_star_state
    deferring their own deletion (see that function's docstring): a husk
    (unstarred, untagged, unarchived) whose entry_unstar_batch row — if any —
    is no longer inside the undo window. Run from nightly maintenance, not
    from the unstar path itself, which is the whole point: it gives an
    accidental bulk unstar in Saved a real window to undo before the entry
    is actually gone, not just the star row. Returns the number deleted.
    """
    saved_url = saved_articles_service.SAVED_FEED_URL
    deleted = 0
    with get_reader() as reader:
        try:
            candidate_ids = [str(e.id) for e in reader.get_entries(feed=saved_url)]
        except Exception:  # noqa: BLE001 — no lectio:saved feed yet, nothing to sweep
            return 0
        for entry_id in candidate_ids:
            if entry_has_keep_signal(saved_url, entry_id, starred=_entry_is_starred(saved_url, entry_id)):
                continue
            with get_meta_connection() as conn:
                row = conn.execute(
                    "SELECT unstarred_at FROM entry_unstar_batch WHERE feed_url = ? AND entry_id = ?",
                    (saved_url, entry_id),
                ).fetchone()
            if row is not None and _undo_token_problem(row["unstarred_at"]) is None:
                continue  # a live undo token still protects this one
            entry = reader.get_entry((saved_url, entry_id), None)
            if entry is None:
                continue
            try:
                _hard_delete_entry(reader, saved_url, entry_id, entry)
                deleted += 1
            except Exception:  # noqa: BLE001 — one bad entry must not sink the sweep
                LOGGER.exception("[maintenance] husk sweep: failed to delete %s", entry_id)
                continue
            with get_meta_connection() as conn:
                conn.execute("DELETE FROM entry_unstar_batch WHERE feed_url = ? AND entry_id = ?", (saved_url, entry_id))
                conn.commit()
    if deleted:
        invalidate_unread_counts_cache()
    return deleted


def restar_curated_entries(feed_url: str) -> int:
    """Star every curated entry in a feed and stamp the star time to now.

    Backs the "show them at the top of the Inbox" option on unsubscribe. The
    Inbox orders by ``saved_entries.saved_at``, so an item curated months ago
    sinks to wherever it was — which is the last place you look right after
    deciding to drop its feed.

    Two different operations, because "re-star" is not one thing:

    - **Tagged but unstarred** — there is no ``saved_entries`` row at all, so a
      date bump would touch nothing and the item would still never appear. It
      has to be genuinely starred, which is also what enqueues its offline
      capture. This is the common case: a tag is a keep signal, but only a star
      is archived.
    - **Already starred** — ``apply_star_state`` is INSERT OR IGNORE, so calling
      it again silently leaves the old timestamp in place and nothing moves. The
      row is re-stamped directly instead.

    Deliberately NOT unstar-then-star for the second case: unstarring an entry
    with no other keep signal enqueues removal of its offline capture, so a
    failure between the two calls would destroy the very thing being preserved.

    Callers must run this BEFORE removing the feed — it reads the feed's entries
    and relies on the capture flush that unsubscribe already performs afterwards.

    Returns the number of entries affected.
    """
    curated: list[str] = []
    with get_reader() as reader:
        for entry in reader.get_entries(feed=feed_url):
            entry_id = str(entry.id)
            if entry_has_keep_signal(feed_url, entry_id, starred=_entry_is_starred(feed_url, entry_id)):
                curated.append(entry_id)

    for entry_id in curated:
        if not _entry_is_starred(feed_url, entry_id):
            apply_star_state(feed_url, entry_id, True)

    if curated:
        with get_meta_connection() as conn:
            conn.executemany(
                "UPDATE saved_entries SET saved_at = CURRENT_TIMESTAMP WHERE feed_url = ? AND entry_id = ?",
                [(feed_url, e) for e in curated],
            )
            conn.commit()
    return len(curated)


def drop_all_curation(feed_url: str) -> dict[str, int]:
    """Strip every keep signal from a feed so unsubscribing really removes it.

    The counterpart to keeping: sometimes a feed was a mistake, and its stars and
    tags are part of the mistake. Without this the only ways out were to keep the
    curation, or to untag and unstar every entry by hand first — because a tag or
    a star preserves the offline capture, so a plain unsubscribe would leave the
    posts behind in Saved as orphan archives.

    Order is load-bearing. Tags come off first: ``apply_star_state(False)``
    consults ``entry_has_keep_signal``, so unstarring while a tag remains would
    (correctly) refuse to release the capture. Archives are then deleted
    explicitly rather than left to ``enqueue_removal``, which only marks the row
    ``pending_removal`` for a background worker — the feed is about to disappear,
    so the cascade has to be synchronous and provable. ``delete_archive`` keeps
    assets that another entry still references.

    ``archived_entries`` (the Archive *state*, not the capture) needs no handling:
    with every capture gone, ``_purge_dead_entry_meta`` treats the whole feed as
    uncaptured and drops all its per-entry meta.

    Returns counts of what was removed.
    """
    counts = {"untagged": 0, "unstarred": 0, "archives": 0}

    entries: list[tuple[str, tuple[str, str]]] = []
    with get_reader() as reader:
        for entry in reader.get_entries(feed=feed_url):
            entries.append((str(entry.id), entry.resource_id))

    for entry_id, resource_id in entries:
        with get_reader() as reader:
            has_tags = bool(get_manual_tags_for_resource(reader, resource_id))
        if has_tags:
            set_manual_tags_for_entry(feed_url, entry_id, "")
            counts["untagged"] += 1

        if _entry_is_starred(feed_url, entry_id):
            apply_star_state(feed_url, entry_id, False)
            counts["unstarred"] += 1

        try:
            if starred_archive_service.delete_archive(feed_url, entry_id):
                counts["archives"] += 1
        except Exception:  # noqa: BLE001
            LOGGER.warning("[drop] archive delete failed for %s/%s", feed_url, entry_id, exc_info=True)

    # Tags an orphan capture picked up while its feed was already gone, plus
    # anything a previous partial removal left; harmless when empty.
    with get_meta_connection() as conn:
        conn.execute("DELETE FROM orphan_entry_tags WHERE feed_url = ?", (feed_url,))
        conn.commit()

    return counts


def mark_entry_read_everywhere(feed_url: str, entry_id: str) -> None:
    """Mark an entry read at both levels, the way a triage action means it.

    reader's own read flag *and* ``entry_read_state`` — the latter is an
    override re-applied on refresh, so marking read in reader alone doesn't
    stick. Also appends to ``read_history``, which is what makes an archived or
    deleted item findable afterwards in the History view.
    """
    try:
        with get_reader() as reader:
            entry = reader.get_entry((feed_url, entry_id), None)
            feed = reader.get_feed(feed_url, None)
            reader.mark_entry_as_read((feed_url, entry_id))
    except Exception:
        entry = feed = None
        LOGGER.warning("mark_entry_read_everywhere: reader write failed for %s/%s", feed_url, entry_id, exc_info=True)
    try:
        upsert_entry_read_state(feed_url, entry_id)
    except Exception:
        LOGGER.warning("mark_entry_read_everywhere: read-state override failed for %s/%s", feed_url, entry_id, exc_info=True)
    try:
        append_read_history(
            feed_url,
            entry_id,
            str(getattr(entry, "title", None) or ""),
            str(getattr(entry, "link", None) or ""),
            str(getattr(feed, "resolved_title", None) or getattr(feed, "title", None) or ""),
        )
    except Exception:
        LOGGER.warning("mark_entry_read_everywhere: history append failed for %s/%s", feed_url, entry_id, exc_info=True)
    invalidate_unread_counts_cache()


def get_saved_unread_count() -> int:
    """Unread count across all kept (starred OR tagged) entries, for the sidebar's
    Saved Articles badge. Kept sets span hundreds of feeds, so this is a single
    set intersection rather than a per-feed or per-chunk query loop, which got
    slower under concurrent refresh writes (home-route gap_block, 2026-09-01)."""
    # Served from a short cache keyed on the unread generation plus a cheap saved_entries signature, so a read-state change, a
    # star or an unstar recomputes at once. Tag writes clear the whole _has_manual_tags_cache, and the TTL bounds anything the
    # signature can't see (a delete and an insert of the newest row between two reads leave the same count and max rowid).
    db_path = str(tenancy.reader_db_path())
    with get_meta_connection() as conn:
        _n, _max_rowid = conn.execute("SELECT COUNT(*), MAX(rowid) FROM saved_entries").fetchone()
    sig = (get_unread_counts_generation(), int(_n or 0), int(_max_rowid or 0), db_path)
    with _has_manual_tags_lock:
        cached = _has_manual_tags_cache.get("saved_unread")
    if cached and cached[0] == sig and time.time() - cached[1] < _TAGGED_KEYS_CACHE_TTL_SECONDS:
        return cached[2]
    count = _compute_saved_unread_count()
    with _has_manual_tags_lock:
        _has_manual_tags_cache["saved_unread"] = (sig, time.time(), count)
    return count


def _compute_saved_unread_count() -> int:
    kept: set[tuple[str, str]] = set()
    with get_meta_connection() as conn:
        for row in conn.execute("SELECT feed_url, entry_id FROM saved_entries"):
            kept.add((str(row["feed_url"]), str(row["entry_id"])))
    kept |= get_tagged_entry_keys()  # union the tag axis in
    if not kept:
        return 0
    # One scan of the unread keys intersected with the kept set, rather than ~30 chunked (feed, id) IN (VALUES ...)
    # probes against a 25k-pair kept set: same count, ~50ms instead of ~250ms on the live library.
    with get_reader() as reader:
        db = reader._storage.get_db()
        return sum(1 for row in db.execute("SELECT feed, id FROM entries WHERE read = 0") if (str(row[0]), str(row[1])) in kept)


def get_starred_inbox_total() -> int:
    """How many items the Saved Inbox holds: every star, minus anything archived.

    Stars only — the same narrowing Read Mode's Inbox uses. The sidebar's other
    Saved counts are *kept* (starred OR tagged) because a tag is filing and the
    tag tree has to count what it lists; the Inbox is a to-do pile, and filing
    something is not a to-do.
    """
    with get_meta_connection() as conn:
        return int(
            conn.execute(
                "SELECT COUNT(*) FROM saved_entries s WHERE NOT EXISTS ("
                " SELECT 1 FROM archived_entries a"
                " WHERE a.feed_url = s.feed_url AND a.entry_id = s.entry_id)"
            ).fetchone()[0]
        )


def get_saved_counts_by_folder(folder_feed_urls_by_id: dict[int, set[str]]) -> dict[int, int]:
    """Total kept (starred OR tagged) entries per folder, for the Saved Articles
    sidebar sublist (the Kept view defaults to All, so totals — the whole backlog
    — are the meaningful number, unlike the unread badges on the feeds tree)."""
    # Union starred and tagged keys so the badge matches what the Kept view shows.
    # Only the folders' own feeds matter, so constrain the tag scan to them rather
    # than scanning the whole entry_tags table.
    relevant_feeds: set[str] = set()
    for urls in folder_feed_urls_by_id.values():
        relevant_feeds |= urls
    kept_by_feed: dict[str, set[str]] = {}
    with get_meta_connection() as conn:
        for row in conn.execute("SELECT feed_url, entry_id FROM saved_entries"):
            feed = str(row["feed_url"])
            if feed in relevant_feeds:
                kept_by_feed.setdefault(feed, set()).add(str(row["entry_id"]))
    for feed, eid in get_tagged_entry_keys(relevant_feeds):
        kept_by_feed.setdefault(feed, set()).add(eid)
    if not kept_by_feed:
        return {}
    counts: dict[int, int] = {}
    for folder_id, urls in folder_feed_urls_by_id.items():
        total = sum(len(ids) for feed, ids in kept_by_feed.items() if feed in urls)
        if total:
            counts[folder_id] = total
    return counts


def merge_orphan_saved_entries(
    posts: list[dict],
    *,
    live_feed_urls: set[str],
    sort_by: str,
    sort_dir: str,
    limit: int,
    only_feed_url: str | None = None,
    archived: bool | None = None,
    search_terms: list[str] | None = None,
    kept_scope: str = "kept",
    capture_failed: bool = False,
) -> list[dict]:
    """Append archive-only saved entries (orphans), then re-sort + clip.

    *capture_failed* narrows orphans to failed or empty captures (see get_capture_failed_keys); live *posts* are already narrowed by the
    caller.

    *archived* (the done-axis) must be applied HERE, not by the caller: this
    re-sorts and re-clips to *limit*, so a caller filtering afterwards sees only
    the archived rows that survived a clip made against the unfiltered list. That
    is what made the Archive view show nothing under some sort orders — a single
    archived post sorted to the bottom and fell outside the window.

    Orphans are kept (starred or tagged) entries whose feed is no longer in any
    folder; their metadata comes entirely from the starred archive. Rendered
    alongside live saved entries so unsubscribing a feed doesn't make its
    saves disappear.

    *kept_scope*, forwarded to get_orphan_saved_entries: "starred" (the Inbox)
    narrows to the star axis alone, same as the live-entry path two lines up
    from this function's callers. Each returned orphan carries its own
    is_starred/manual_tags — used below instead of the old hardcoded
    saved=True, which made a tagged-then-unstarred orphan look identically
    starred to a real star in the list (disagreeing with the entry pane) and,
    without this scope, kept it pinned in the Inbox forever.

    When *only_feed_url* is given, restrict orphans to that single feed (matched
    canonically) — used when the user clicks the feed link of an orphaned save
    to browse just that unsubscribed feed's archived items.

    *search_terms*, if given, is forwarded to the archive-metadata match (see
    get_orphan_saved_entries) — the caller's search doesn't otherwise reach
    orphans at all, since they have no reader row for the SQL search to join
    against.
    """
    archived_keys = get_archived_saved_keys() if archived is not None else set()
    if archived is not None:
        posts = [p for p in posts if ((str(p["feed_url"]), str(p["id"])) in archived_keys) == archived]

    orphans = starred_archive_service.get_orphan_saved_entries(live_feed_urls, search_terms, kept_scope=kept_scope)
    if archived is not None:
        orphans = [o for o in orphans if ((str(o["feed_url"]), str(o["id"])) in archived_keys) == archived]
    if capture_failed:
        failed_keys = starred_archive_service.get_capture_failed_keys()
        orphans = [o for o in orphans if (str(o["feed_url"]), str(o["id"])) in failed_keys]
    if only_feed_url is not None:
        target = normalize_feed_url(only_feed_url)
        orphans = [o for o in orphans if normalize_feed_url(o["feed_url"]) == target]
    if not orphans:
        return posts

    sort_desc = normalize_sort_dir(sort_dir) == "desc"
    # Reader path: this runs under resolve_reader_backlog, so the Inbox's
    # star-date order is a legitimate value here.
    normalized_orphan_sort = normalize_sort_by(sort_by, allow_starred=True)

    def _sort_value_from_epoch(epoch: float | None) -> str:
        # Mirrors datetime_sort_value — ISO-format string, empty for None.
        if epoch is None:
            return ""
        try:
            return datetime.fromtimestamp(float(epoch), tz=timezone.utc).isoformat()
        except OverflowError, OSError, ValueError:
            return ""

    existing_keys = {(p["feed_url"], p["id"]) for p in posts}
    sort_key = {
        "post": "post_sort_value",
        "starred": "saved_sort_value",
        "size": "size_sort_value",
    }.get(normalized_orphan_sort, "received_sort_value")

    additions: list[dict] = []
    for orphan in orphans:
        key = (orphan["feed_url"], orphan["id"])
        if key in existing_keys:
            continue
        post_iso = _sort_value_from_epoch(orphan.get("published_at"))
        recv_iso = _sort_value_from_epoch(orphan.get("received_at"))
        additions.append(
            {
                "feed_url": orphan["feed_url"],
                "id": orphan["id"],
                "title": orphan["title"],
                "link": html_sanitize.safe_link_url(orphan["link"]),
                "thumbnail_url": None,
                "feed_title": orphan["feed_title"],
                "feed_icon_url": None,
                "manual_tags": orphan.get("manual_tags") or [],
                "kept": True,
                "read": True,
                "saved": bool(orphan.get("is_starred")),
                "post_sort_value": post_iso,
                "received_sort_value": recv_iso,
                "saved_sort_value": _sort_value_from_epoch(orphan.get("starred_at")),
                "size_sort_value": float(orphan.get("content_size_bytes") or 0),
                "history_sort_value": "",
                "post_timestamp": post_iso or None,
                "received_timestamp": recv_iso or None,
                "read_timestamp": None,
                "post_display": format_datetime_for_ui(
                    datetime.fromtimestamp(orphan["published_at"], tz=timezone.utc) if orphan.get("published_at") else None
                ),
                "received_display": format_datetime_for_ui(
                    datetime.fromtimestamp(orphan["received_at"], tz=timezone.utc) if orphan.get("received_at") else None
                ),
                "read_display": None,
                "size_bytes": orphan.get("content_size_bytes"),
                "size_display": (_format_size_bytes(_ob) if (_ob := orphan.get("content_size_bytes")) is not None else None),
                "duration_seconds": None,
                "duration_display": None,
                "is_orphan_archive": True,
            }
        )

    # Re-augment existing posts with sort_value strings (list_entries_for_feeds
    # popped them before returning); use ISO strings derived from the timestamps
    # the template path keeps around.
    for p in posts:
        if "post_sort_value" not in p:
            p["post_sort_value"] = p.get("post_timestamp") or ""
        if "received_sort_value" not in p:
            p["received_sort_value"] = p.get("received_timestamp") or ""
        if "saved_sort_value" not in p:
            p["saved_sort_value"] = p.get("saved_timestamp") or ""
        if "size_sort_value" not in p:
            # list_entries_for_feeds pops the sort_key it used, same as the ISO
            # fields above, but "size_bytes" (the raw int, used for the display
            # badge) survives — re-derive from that instead of a timestamp.
            p["size_sort_value"] = float(p.get("size_bytes") or 0)

    combined = posts + additions
    # size_sort_value is a float (0 for "nothing archived yet"), not an empty
    # string -- `or ""` here would have coerced a genuine 0-byte/unarchived
    # item into a string and made it uncomparable against the rest.
    default_sort_value = 0.0 if sort_key == "size_sort_value" else ""
    combined.sort(key=lambda item: item.get(sort_key, default_sort_value), reverse=sort_desc)
    combined = combined[:limit]

    for p in combined:
        p.pop("post_sort_value", None)
        p.pop("received_sort_value", None)
        p.pop("saved_sort_value", None)
        p.pop("size_sort_value", None)
        p.pop("history_sort_value", None)

    return combined


def filter_feed_urls(feed_urls: set[str], list_feed_url: str | None) -> set[str]:
    if not list_feed_url:
        return feed_urls
    if list_feed_url in feed_urls:
        return {list_feed_url}
    # The Saved Articles feed is synthetic and absent from get_all_feed_urls, so
    # it would never match above — but it is directly viewable (the File-Saved
    # "review this host" link opens it, scoped by site:). Allow it explicitly.
    if list_feed_url == saved_articles_service.SAVED_FEED_URL:
        return {list_feed_url}
    return set()


def _build_orphan_entry_detail(feed_url: str, entry_id: str) -> dict | None:
    """Render-shaped dict for a starred entry whose feed is gone from the reader.

    Sourced entirely from the starred archive — used as the fallback when
    `reader.get_entry()` returns None. Returns None if the archive has no
    completed row for this entry.
    """
    archived = starred_archive_service.get_archived_entry_detail(feed_url, entry_id)
    if not archived:
        return None
    asset_map = starred_archive_service.get_entry_asset_map(feed_url, entry_id)
    content_html = archived.get("content_html")
    if isinstance(content_html, str) and content_html and asset_map:
        content_html = starred_archive_service.rewrite_html_assets(content_html, asset_map, STARRED_ASSET_URL_PREFIX)

    def _fmt(epoch: float | None) -> str | None:
        if epoch is None:
            return None
        try:
            dt = datetime.fromtimestamp(float(epoch), tz=timezone.utc)
        except OverflowError, OSError, ValueError:
            return None
        return format_datetime_for_ui(dt)

    def _iso(epoch: float | None) -> str | None:
        if epoch is None:
            return None
        try:
            return datetime.fromtimestamp(float(epoch), tz=timezone.utc).isoformat()
        except OverflowError, OSError, ValueError:
            return None

    published_at = archived.get("published_at")
    received_at = archived.get("received_at")
    # Orphan archives can't carry reader-backed manual tags (no reader
    # resource — see set_manual_tags_for_entry), but they still support
    # tagging via orphan_entry_tags, same as Star already does via saved_entries.
    manual_tags = _get_orphan_manual_tags(feed_url, entry_id)
    is_starred = _entry_is_starred(feed_url, entry_id)
    # A surviving capture is not itself a keep signal — same star-OR-tag rule
    # as every live entry (see get_orphan_saved_entries). Previously hardcoded
    # True here, so an orphan with neither a star nor a tag (a leftover from
    # before this table existed, or unstarred after the feed was already gone)
    # looked identically "kept" to one you'd actually curated.
    is_kept = is_starred or bool(manual_tags)
    # Orphans have no reader resource, so there's no feed-provided (publisher)
    # tags to merge in here (see the manual_tags comment above) — but the
    # user's own pinned/suggested tags (Feed Properties, meta-DB only) are
    # exactly as available as they are for a live entry. This was hardcoded to
    # [] before, so an orphan feed's suggestion chip never showed no matter
    # what was pinned to it. Same "already applied -> stop suggesting" rule
    # as the live path (routes.entries.entry_pane).
    _manual_now = {normalize_tag_value(t) for t in manual_tags}
    feed_tag_suggestions = [t for t in get_feed_pinned_tags(feed_url) if t not in _manual_now]

    return {
        "feed_url": feed_url,
        "id": entry_id,
        "title": archived.get("title") or "",
        # Archived copies carry the link the feed gave us — same guard as the
        # live path (see safe_link_url).
        "link": html_sanitize.safe_link_url(archived.get("link")),
        "summary": "",
        "content_html": content_html,
        "lead_image_url": None,
        "katex_dollar_math": False,
        "image_title_text": None,
        "duration_seconds": None,
        "duration_display": None,
        "feed_title": archived.get("feed_title") or feed_url,
        "post_timestamp": _iso(published_at),
        "received_timestamp": _iso(received_at),
        "post_display": _fmt(published_at),
        "received_display": _fmt(received_at),
        "author": archived.get("author"),
        "read": True,
        "saved": is_starred,
        "kept": is_kept,
        "manual_tags": manual_tags,
        "manual_tags_text": " ".join(manual_tags),
        "feed_tag_suggestions": feed_tag_suggestions,
        "feed_tag_chips_collapsed": FEED_TAG_CHIPS_COLLAPSED,
        "feed_tag_filter_signs": {},
        "feed_tag_inherited_signs": {},
        "author_filter_token": None,
        "feed_icon_url": None,
        "is_orphan_archive": True,
    }


def _rewrite_url_host(url: str | None, aliases: dict[str, str]) -> str | None:
    """Rewrite *url*'s host through a ``{from_host: to_host}`` alias map (a
    leading ``www.`` folds). Returns *url* unchanged when nothing matches."""
    if not url or not aliases:
        return url
    parsed = urlparse(str(url))
    host = parsed.netloc.lower()
    bare = host[4:] if host.startswith("www.") else host
    new_host = aliases.get(bare)
    if not new_host or new_host == parsed.netloc:
        return url
    return parsed._replace(netloc=new_host).geturl()


def _rebase_proxy_entry_link(entry_link: str | None, feed_url: str, channel_link: str | None) -> str | None:
    """Rebase an entry link that points to a proxy host (e.g. feedburner) back to the
    real publisher host stored in the feed's channel link element.

    The caller folds ``channel_link`` through the feed's declared host migrations
    first (see `_rewrite_url_host` / `get_dedupe_host_aliases`): a feed whose
    channel ``<link>`` still names the author's dead domain would otherwise rebase
    already-correct entry links back onto it — the tush.ar/tushar.lol case."""
    if not entry_link or not channel_link:
        return entry_link
    ep = urlparse(str(entry_link))
    fp = urlparse(str(feed_url))
    cp = urlparse(str(channel_link))
    if ep.netloc and ep.netloc == fp.netloc and cp.netloc and cp.netloc != fp.netloc:
        return (
            cp.scheme + "://" + cp.netloc + ep.path + (("?" + ep.query) if ep.query else "") + (("#" + ep.fragment) if ep.fragment else "")
        )
    return entry_link


def _lead_image_display_url(image_url: str | None, size_rule: str | None = None) -> str | None:
    """Return the URL to use in the browser for a lead image.

    *size_rule* is the feed's optional ``"<param>=<value>"`` image-size upgrade
    (see ``upgrade_image_size_param``), applied before proxying so the proxy
    caches the size actually shown rather than the thumbnail it replaced.

    Some sites set Cross-Origin-Resource-Policy: same-site on their images,
    which modern browsers enforce for cross-origin <img> loads — the image
    appears as a broken icon even though the server returns HTTP 200.
    For such domains we route the image through our server-side proxy so the
    browser only ever sees a same-origin request.

    Results are cached in _CORP_DOMAIN_CACHE (per process, per domain) so only
    the first new domain incurs a HEAD request.
    """
    if not image_url:
        return None
    image_url = upgrade_image_size_param(image_url, size_rule)
    domain = urlparse(image_url).netloc
    # Hotlink-protected hosts: proxy unconditionally (the placeholder is served
    # with HTTP 200, so there's nothing to detect — the host is known-bad).
    if _is_hotlink_img_host(domain):
        return f"/api/img?u={quote(image_url)}"
    if domain not in _CORP_DOMAIN_CACHE:
        # Optimistically return the direct URL and check CORP in the background.
        # First render may show a broken image for CORP-gated domains; subsequent
        # renders proxy correctly once the cache is warm.
        def _check_corp(url: str = image_url, d: str = domain) -> None:
            try:
                resp = httpx.head(url, follow_redirects=True, timeout=3.0, headers={"User-Agent": READABILITY_USER_AGENT})
                corp = resp.headers.get("cross-origin-resource-policy", "").strip().lower()
                _CORP_DOMAIN_CACHE[d] = corp in ("same-site", "same-origin")
            except Exception:
                _CORP_DOMAIN_CACHE[d] = False

        threading.Thread(target=_check_corp, daemon=True).start()
        return image_url
    if _CORP_DOMAIN_CACHE.get(domain):
        return f"/api/img?u={quote(image_url)}"
    return image_url


def _derive_article_lead_image(entry) -> str | None:
    """Pick the article lead image, honoring an inline / media_rss feed strategy.

    Those strategies derive the image straight from the entry's own content or
    RSS fields and deliberately bypass the lead-image cache — the same extractors
    the post-list thumbnail uses. The article view previously always called the
    cache-consulting ``extract_entry_thumbnail_url``, so a stale negative cache
    entry (image resolved as "none" before the feed was pinned to inline, e.g. a
    DeviantArt gallery) made the article show no image while the list still
    showed the inline thumbnail. Routing by strategy keeps the two consistent."""
    feed_url = str(getattr(entry, "feed_url", "") or "")
    strategy, _, _ = lead_image_service.get_feed_strategy(feed_url)
    # Each strategy has a primary extractor that bypasses the lead-image cache
    # (so the article matches the list thumbnail's own logic):
    #   inline / webcomic → inline <img> (claycomix ships the full strip inline;
    #                       DeviantArt galleries dodge a stale negative cache)
    #   media_rss         → <media:content> (reader keeps media:thumbnail/enclosure)
    # Every strategy then falls back to the cache-consulting extractor — covering
    # feeds whose image lives in an <enclosure>/media field reader dropped (gottadeal,
    # paizo) — and finally to a raw inline <svg>. Without the cache fallback the
    # article showed no image AND persisted a negative on open, poisoning the thumb.
    if strategy in ("inline", "webcomic"):
        primary = lead_image_service.extract_inline_thumb_url(entry)
        if strategy == "webcomic":
            # ComicControl-style feeds (atomic-robo, smbc, misfile) ship only a
            # small /comicsthumbs/ image inline, while the full panel is what the
            # source-page scan already cached — so the inline-first rule above
            # showed the thumbnail and the article never got the readable comic.
            # Prefer a cached *positive*; a cached negative still loses to the
            # inline image, which is the stale-negative case this bypass exists
            # for (claycomix ships the strip inline and is unaffected either way).
            _cached_full = lead_image_service.get_cached_lead_image_url(feed_url, str(getattr(entry, "id", "") or ""))
            if _cached_full:
                primary = _cached_full
    elif strategy == "media_rss":
        primary = lead_image_service.extract_media_rss_thumb_url(entry)
    else:
        primary = None
    return (
        primary
        or lead_image_service.extract_entry_thumbnail_url(entry, include_source_lookup=False)
        or lead_image_service.extract_inline_svg_thumb_url(entry)
    )


_PLAINTEXT_PROMOTE_RE = re.compile(r"https?://|&lt;br|<br", re.IGNORECASE)
_BARE_URL_RE = re.compile(r"https?://[^\s<>\"']+")
# A bare URL that points straight at an image — rendered inline as <img> rather
# than a link when it appears in a promoted plain-text summary. Trailing
# punctuation is excluded by _BARE_URL_RE's character class already.
_BARE_IMG_URL_RE = re.compile(r"\.(?:png|jpe?g|gif|webp|bmp|svg)$", re.IGNORECASE)


# A real HTML tag (block or inline) — used to distinguish genuine HTML content
# from plain text that a feed has merely escaped and mislabeled as text/html.
_REAL_HTML_TAG_RE = re.compile(
    r"<(?:p|div|br|a|img|ul|ol|li|table|tr|td|blockquote|pre|code|h[1-6]|span|strong|em|b|i)\b",
    re.IGNORECASE,
)


def _looks_like_escaped_plaintext(value: str | None) -> bool:
    """True when content declared as HTML is really escaped plain text: it carries
    escaped break markers (``&lt;br&gt;``) but no actual HTML tags. Such feeds
    (e.g. tracker.example news) otherwise render as inert escaped text."""
    if not value:
        return False
    if "&lt;br" not in value.lower():
        return False
    return not _REAL_HTML_TAG_RE.search(value)


def _linkify_plaintext_description(text: str) -> str:
    """Render a plain-text description (YouTube) as HTML: escape once, linkify bare URLs, keep line breaks.

    The input can arrive HTML-escaped ("Sources &amp; further reading"), so it is unescaped first. URLs are matched on the unescaped text,
    where the regex really does exclude spaces, quotes and angle brackets, and each piece is escaped exactly once. Trailing sentence
    punctuation and an unbalanced ")" are left outside the link, so "(https://example.com/x)" links to just the URL."""
    text = html.unescape(text).replace("\r\n", "\n").replace("\r", "\n")
    out: list[str] = []
    pos = 0
    for m in _BARE_URL_RE.finditer(text):
        url, tail = m.group(0), ""
        while url and (url[-1] in ".,!?:" or (url[-1] == ")" and url.count(")") > url.count("("))):
            tail = url[-1] + tail
            url = url[:-1]
        out.append(html.escape(text[pos : m.start()]))
        safe_url = html.escape(url)
        out.append(f'<a href="{safe_url}" target="_blank" rel="noopener noreferrer">{safe_url}</a>{html.escape(tail)}')
        pos = m.end()
    out.append(html.escape(text[pos:]))
    return "".join(out).replace("\n", "<br>")


def _promote_plaintext_summary(summary: str | None) -> str | None:
    """Turn a bare-text summary into renderable HTML, or None to leave it as-is.

    Some feeds (e.g. tracker.example news) ship no HTML content — only a
    plain-text summary carrying bare ``https://`` URLs and line breaks encoded as
    literal ``<br>`` or double-escaped ``&lt;br&gt;``. Rendered in the template's
    ``<pre>`` fallback those URLs aren't clickable and the breaks show as literal
    text. When a summary actually contains URLs or break markers, promote it to
    content_html: normalize the breaks to real ``<br>`` and linkify the URLs.
    Genuinely plain prose (no URLs, no breaks) returns None so the ``<pre>``
    fallback keeps preserving its whitespace layout."""
    if not summary or not summary.strip():
        return None
    if not _PLAINTEXT_PROMOTE_RE.search(summary):
        return None
    # Collapse escaped/literal <br> (possibly multiply-escaped, e.g. &amp;lt;br&amp;gt;)
    # to newlines, then treat the whole thing as plain text.
    text = re.sub(r"&(?:amp;)*lt;br\s*/?\s*&(?:amp;)*gt;", "\n", summary, flags=re.IGNORECASE)
    text = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)
    # These feeds escape inconsistently — break markers once but ampersands in
    # URLs twice (&amp;amp;). Fully unescape (bounded) so URLs are correct, then
    # re-escape exactly once below. Breaks are already newlines, so unescaping
    # any remaining &lt;…&gt; just yields inert text we re-escape anyway.
    for _ in range(3):
        unescaped = html.unescape(text)
        if unescaped == text:
            break
        text = unescaped
    # Tame runs of blank lines (the feed pairs each <br> with a real newline).
    text = re.sub(r"\n{3,}", "\n\n", text)
    escaped = html.escape(text)

    def _linkify(m: re.Match) -> str:
        seg = m.group(0)  # already HTML-escaped; src/href-safe (no spaces/quotes)
        if _BARE_IMG_URL_RE.search(html.unescape(seg)):
            return f'<img src="{seg}" alt="" loading="lazy" referrerpolicy="no-referrer" style="max-width:100%;height:auto;">'
        return f'<a href="{seg}" target="_blank" rel="noopener noreferrer">{seg}</a>'

    escaped = _BARE_URL_RE.sub(_linkify, escaped)
    return escaped.replace("\n", "<br>")


def _youtube_embed_html(video_id: str) -> str:
    """Inline YouTube player markup for a video id.

    Host follows the per-user privacy/features preference (``youtube_embed_host``):
    privacy-enhanced ``youtube-nocookie.com`` by default, or the standard
    ``www.youtube.com`` (Share / Watch Later) when the user opts in. Uses
    ``referrerpolicy`` rather than the JS API. We deliberately omit
    ``enablejsapi=1`` — nothing in the app drives the IFrame JS API, and YouTube
    refuses playback when it is set without a matching ``origin=`` parameter,
    which silently broke the inline player.

    The result is injected into content_html (rendered with ``| safe``), so the
    video id is HTML-escaped before interpolation — defense in depth in case the
    upstream extractor's validation is ever loosened (the id is otherwise always
    ``[A-Za-z0-9_-]``)."""
    safe_id = html.escape(video_id, quote=True)
    src = f"https://{youtube_embed_host()}/embed/{safe_id}?rel=0"
    return (
        '<div class="youtube-embed-container" style="max-width:560px;margin:1em auto;">'
        f'<iframe width="100%" height="315" src="{src}" title="YouTube video player" '
        'frameborder="0" allowfullscreen loading="lazy" '
        'referrerpolicy="strict-origin-when-cross-origin" '
        'allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"'
        "></iframe></div>"
    )


# Matches any iframe's src value, NOT just Bandcamp's — the marker test moved
# into _fix below. Embedding the literal between two `[^"\']*` stars (the earlier
# form) made the split point ambiguous, so a src carrying many repetitions of
# `bandcamp.com/EmbeddedPlayer/` backtracked quadratically: 250 reps ≈ 19ms,
# 2,000 ≈ 1.2s, and feed HTML is attacker-controlled. One unambiguous star has
# no split to search.
_BC_EMBED_IFRAME_RE = re.compile(
    r'(<iframe\b[^>]*\bsrc=["\'])([^"\']*)(["\'])',
    re.I,
)
_BC_EMBED_SRC_MARKER = "bandcamp.com/embeddedplayer/"


def _strip_bandcamp_track_signature(content_html: str) -> str:
    """Drop the domain-locked single-track signature from Bandcamp embeds.

    Bandcamp's ``.../tracks=<ids>/esig=<sig>/`` single-track player form is bound
    to the publisher's domain — Bandcamp validates the Referer, so it renders
    "Sorry, this track or album is not available." anywhere else (including
    Lectio). The plain ``album=<id>`` form embeds on any site and plays the same
    (pre-order/premiere) album, so strip the ``tracks``/``esig`` path segments and
    fall back to it rather than show a dead player."""
    if not isinstance(content_html, str) or "bandcamp.com/EmbeddedPlayer" not in content_html:
        return content_html
    if "tracks=" not in content_html and "esig=" not in content_html:
        return content_html

    def _fix(m: re.Match) -> str:
        src = m.group(2)
        # The regex matches every iframe src; only Bandcamp embeds are rewritten
        # (this test used to live in the pattern, where it cost quadratic
        # backtracking — see _BC_EMBED_IFRAME_RE).
        if _BC_EMBED_SRC_MARKER not in src.lower():
            return m.group(0)
        src = re.sub(r"/tracks=[\w,]+", "", src)
        src = re.sub(r"/esig=[0-9a-f]+", "", src, flags=re.IGNORECASE)
        return m.group(1) + src + m.group(3)

    return _BC_EMBED_IFRAME_RE.sub(_fix, content_html)


_YT_EMBED_SRC_HOST_RE = re.compile(
    r'(?P<pre><iframe\b[^>]*\bsrc=["\']https://)'
    r"(?:www\.)?youtube(?:-nocookie)?\.com"
    r"(?P<post>/embed/)",
    re.IGNORECASE,
)


def _apply_youtube_embed_host(content_html: str) -> str:
    """Rewrite feed-native YouTube embed iframe hosts to the current user's
    privacy/features preference (youtube.com vs youtube-nocookie.com). Only
    touches iframe /embed/ URLs, not plain youtube.com watch links."""
    if not isinstance(content_html, str) or "/embed/" not in content_html:
        return content_html
    host = youtube_embed_host()
    return _YT_EMBED_SRC_HOST_RE.sub(rf"\g<pre>{host}\g<post>", content_html)


_YT_EMBED_FIGURE_CLASS_RE = re.compile(r"is-provider-youtube|wp-block-embed-youtube", re.I)
# Empty placeholders left behind when an embed <iframe> was stripped at ingest:
# WordPress' provider figure, or ArtStation's video-wrapper div.
_YT_EMBED_PLACEHOLDER_RE = re.compile(r"is-provider-youtube|wp-block-embed-youtube|video-wrapper", re.I)


def _inject_recovered_youtube_embeds(content_html: str, video_ids: list[str]) -> str:
    """Replace stripped YouTube embed placeholders with real players.

    feedparser/ingest removes the embed ``<iframe>``, leaving an empty shell:
    WordPress' ``<figure class="wp-block-embed ... is-provider-youtube">`` or
    ArtStation's ``<div class="video-wrapper media-asset...">``. Replace each such
    placeholder, in document order, with the player for the matching recovered
    video id. Placeholders past the end of ``video_ids`` are left untouched."""
    if not video_ids:
        return content_html
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(content_html, "html.parser")
    placeholders = []
    for tag in soup.find_all(["figure", "div"]):
        classes = " ".join(tag.get("class") or [])  # type: ignore[arg-type]
        if tag.name == "figure" and _YT_EMBED_FIGURE_CLASS_RE.search(classes):
            placeholders.append(tag)
        elif tag.name == "div" and "video-wrapper" in classes.lower() and not tag.find(["iframe", "video"]):
            placeholders.append(tag)
    if not placeholders:
        return content_html
    for ph, vid in zip(placeholders, video_ids, strict=False):
        ph.replace_with(BeautifulSoup(_youtube_embed_html(vid), "html.parser"))
    return str(soup)


# /embed/ is included because publishers link it as well as iframe it. Sonarsource
# ships its post's video as <a href="https://www.youtube.com/embed/<id>?si=…">, a
# paragraph on its own — an embed URL wearing an anchor. Following it works, but
# the video sat on the site and not in the article, which is exactly what this
# function exists to undo. youtube-nocookie is the same URL shape from the
# privacy host and would otherwise be missed for no reason.
_YT_WATCH_URL_RE = re.compile(
    r"^(?:https?://)?(?:www\.|m\.)?(?:youtube\.com/watch\?[^\s]*\bv=|youtu\.be/|"
    r"youtube\.com/shorts/|(?:youtube|youtube-nocookie)\.com/embed/)[\w-]+",
    re.IGNORECASE,
)

_BC_URL_RE = re.compile(
    r"^https?://[^/]*\.bandcamp\.com/(album|track)/[^/?#\s]+",
    re.IGNORECASE,
)


# Inline formatting tags a citation-style link is commonly wrapped in — these
# don't make a link "alone" the way sitting directly in a bare <p> does.
# "Inigo Montoya once said, <em><a href=youtu.be/...>quote</a></em>." has
# <em>, not <p>, as the anchor's immediate parent, so a check that only ever
# looked at the immediate parent silently skipped verifying the paragraph's
# other text existed — converting a citation link buried in prose into a
# full embedded player. Root-caused 2026-08-30 on exactly that shape.
_INLINE_WRAPPER_TAGS = {"em", "strong", "b", "i", "span", "small", "mark", "u", "s", "sub", "sup", "cite", "q", "abbr"}
# Containers a standalone link's text is compared against once inline
# wrappers are climbed past. <p> was the only one recognized at first —
# raised in review 2026-08-31: a link wrapped for emphasis inside a <div> or
# <li> (rather than a <p>) fell through to comparing against just the inline
# wrapper itself, so other prose in that block went undetected and the link
# still converted, losing the citation text exactly like the <p> case this
# function was built to fix.
_BLOCK_CONTAINER_TAGS = {
    "p",
    "div",
    "li",
    "td",
    "th",
    "blockquote",
    "section",
    "article",
    "aside",
    "figure",
    "figcaption",
    "header",
    "footer",
    "main",
}


def _standalone_link_target(a):
    """The element to replace with an embed if *a* is the sole meaningful
    content of its block, else None (a worded link sitting inside prose).

    Walks up through inline formatting wrappers to the first block-level
    ancestor (or the anchor itself, if there is none) before comparing text,
    so a link wrapped in <em>/<strong>/etc. still gets the same "nothing else
    in this block" check a bare <p><a>...</a></p> gets — an inline wrapper
    must not exempt a link from it, only change which node holds the text to
    compare against."""
    anchor_text = a.get_text(strip=True)
    node = a
    parent = a.parent
    while parent is not None and parent.name in _INLINE_WRAPPER_TAGS:
        node = parent
        parent = parent.parent
    if parent is not None and parent.name in _BLOCK_CONTAINER_TAGS:
        if parent.get_text(strip=True) != anchor_text:
            return None  # other prose in the block → inline mention
        return parent
    if node.get_text(strip=True) != anchor_text:
        return None  # other content alongside the link's wrapper → inline mention
    return node


def _embed_standalone_youtube_links(content_html: str) -> str:
    """Turn a paragraph/anchor that is *only* a bare YouTube link into a player.

    Some feeds (WordPress with the oEmbed iframe stripped) deliver the video as
    a standalone watch/youtu.be/shorts link in its own paragraph rather than an
    embed. When a <p> (or a lone <a>) contains nothing but such a link, replace
    it with the inline player. Inline mentions inside prose are left untouched —
    we only convert when the link is the element's sole meaningful content."""
    if not isinstance(content_html, str) or "youtu" not in content_html.lower():
        return content_html
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(content_html, "html.parser")
    changed = False
    for a in list(soup.find_all("a")):
        href = str(a.get("href") or "").strip()
        if not href or not _YT_WATCH_URL_RE.match(href):
            continue
        target = _standalone_link_target(a)
        if target is None:
            continue
        vid = youtube_duration_service.extract_video_id(href)
        if not vid:
            continue
        target.replace_with(BeautifulSoup(_youtube_embed_html(vid), "html.parser"))
        changed = True
    return str(soup) if changed else content_html


def _extract_bc_numeric_id(page_html: str, embed_type: str) -> str | None:
    """Extract the numeric Bandcamp album or track ID from page HTML.

    Tries the EmbeddedPlayer URL (present in og:video and inline scripts) then
    a data-album-id / data-track-id attribute as fallback."""
    m = re.search(rf"EmbeddedPlayer/{embed_type}=(\d+)", page_html, re.IGNORECASE)
    if m:
        return m.group(1)
    m = re.search(rf'data-{embed_type}-id=["\'](\d+)["\']', page_html)
    if m:
        return m.group(1)
    return None


def _bc_embed_html(embed_type: str, numeric_id: str, album_url: str) -> str:
    height = "120px" if embed_type == "album" else "42px"
    src = (
        f"https://bandcamp.com/EmbeddedPlayer/{embed_type}={numeric_id}"
        "/size=large/bgcol=ffffff/linkcol=0687f5/tracklist=false/transparent=true/"
    )
    safe_url = html.escape(album_url, quote=True)
    return (
        f'<p class="lectio-embed">'
        f'<iframe style="border:0;width:100%;height:{height}"'
        f' src="{src}" seamless>'
        f'<a href="{safe_url}">{html.escape(album_url)}</a>'
        f"</iframe></p>"
    )


def _embed_standalone_bandcamp_links(content_html: str) -> str:
    """Turn a lone bare Bandcamp album/track link in its own paragraph into an embed.

    Mirrors _embed_standalone_youtube_links but requires the album page to be
    fetched first (the numeric ID is not in the public URL). Cache-first: if
    the album page is already in _source_html_cache the embed is inlined on
    this open; otherwise a background fetch is queued so it appears next open."""
    if not isinstance(content_html, str) or "bandcamp.com" not in content_html.lower():
        return content_html
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(content_html, "html.parser")
    changed = False
    for a in list(soup.find_all("a", href=True)):
        href = str(a.get("href") or "").strip()
        m = _BC_URL_RE.match(href)
        if not m:
            continue
        embed_type = m.group(1).lower()  # "album" or "track"
        target = _standalone_link_target(a)
        if target is None:
            continue
        cached = lead_image_service.get_cached_source_html(href)
        if cached:
            _, page_html = cached
            numeric_id = _extract_bc_numeric_id(page_html, embed_type)
            if numeric_id:
                target.replace_with(BeautifulSoup(_bc_embed_html(embed_type, numeric_id, href), "html.parser"))
                changed = True
                continue
        # Album page not cached or ID not extractable — queue background fetch;
        # embed resolves on the next open once the fetch populates the cache.
        lead_image_service.queue_source_html_fetch(href)
    return str(soup) if changed else content_html


def _extract_source_embed_iframes(raw_html: str, existing_html: str = "", limit: int = 8) -> list[tuple[str | None, str]]:
    """Pull allowlisted media-embed players out of a source page's raw HTML.

    Returns ``(canonical_link, embed_html)`` pairs in document order, skipping any
    whose src is already present in ``existing_html`` (deduped on the src path).
    YouTube embeds are rebuilt via ``_youtube_embed_html`` so they honor the
    user's host preference; every other embed is run through the shared sanitizer
    (sandbox + referrer policy) since it's injected at render time. The canonical
    link (``yt:<id>`` for YouTube, else the embed's fallback ``<a href>`` so a
    Bandcamp/SoundCloud player can be matched to its bare album/track link in the
    body) drives in-context placement; ``None`` when no link can be derived.

    Besides real ``<iframe>``s, recognizes click-to-load YouTube **facades**
    (e.g. guitarworld.com): pages whose JS swaps a thumbnail for the iframe on
    click, so the raw HTML only carries a ``/vi/<id>/`` thumbnail image inside
    a video-ish container, or a ``<lite-youtube videoid=…>`` element."""
    if not raw_html:
        return []
    lower_html = raw_html.lower()
    has_iframe = "<iframe" in lower_html
    has_facade = "ytimg.com/vi/" in lower_html or "youtube.com/vi/" in lower_html or "lite-youtube" in lower_html
    if not (has_iframe or has_facade):
        return []
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(raw_html, "html.parser")
    existing = (existing_html or "").lower()
    seen: set[str] = set()
    seen_yt: set[str] = set()
    out: list[tuple[str | None, str]] = []

    def _emit_youtube(vid: str) -> None:
        # `existing` is lowercased, so lower the needle too (IDs are
        # case-sensitive, but two same-page IDs differing only by case don't
        # happen in practice, and erring toward skip is harmless here).
        if vid in seen_yt or f"/embed/{vid}".lower() in existing:
            return
        seen_yt.add(vid)
        out.append((f"yt:{vid}", _youtube_embed_html(vid)))

    if has_iframe:
        for ifr in soup.find_all("iframe"):
            if len(out) >= limit:
                break
            src = next((v for v in (str(ifr.get(a) or "").strip() for a in ("src", "data-src", "data-lazy-src")) if v), "")
            if not src or not html_sanitize._embed_host_allowed(src):
                continue
            key = src.split("?", 1)[0].lower()
            if key in seen or key in existing:
                continue
            seen.add(key)
            vid = None
            if "youtu" in src.lower():
                _em = re.search(r"/embed/([\w-]{11})", src)
                vid = _em.group(1) if _em else youtube_duration_service.extract_video_id(src)
            if vid:
                _emit_youtube(vid)
            else:
                cleaned = html_sanitize.sanitize_html(str(ifr))
                if "<iframe" in cleaned.lower():
                    # iframe content is parsed as raw text (the fallback <a> is a
                    # string, not a tag), so match its href out of the inner markup —
                    # the canonical album/track link for matching the body's bare link.
                    _am = re.search(r'href=["\']([^"\']+)["\']', ifr.decode_contents())
                    canonical = _am.group(1) if _am else None
                    out.append((canonical, cleaned))

    if has_facade:
        # <lite-youtube videoid="…"> custom elements (the common facade library).
        for el in soup.find_all("lite-youtube"):
            if len(out) >= limit:
                break
            vid = str(el.get("videoid") or "").strip()
            if re.fullmatch(r"[\w-]{11}", vid):
                _emit_youtube(vid)
        # Facade thumbnails: an <img> pointing at the video's /vi/<id>/ thumb.
        # Require a video-ish ancestor class/id so unrelated "related videos"
        # thumbnails elsewhere on the page don't get pulled in.
        for img in soup.find_all("img"):
            if len(out) >= limit:
                break
            src = str(img.get("src") or img.get("data-src") or "")
            m = re.search(r"(?:ytimg\.com|youtube\.com)/vi/([\w-]{11})/", src, re.IGNORECASE)
            if not m:
                continue
            node, hinted = img, False
            for _ in range(4):
                node = node.parent
                if node is None:
                    break
                blob = " ".join([*(node.get("class") or []), str(node.get("id") or "")]).lower()
                if any(h in blob for h in ("youtube", "video", "facade", "embed", "player")):
                    hinted = True
                    break
            if hinted:
                _emit_youtube(m.group(1))
    return out


def _norm_media_link(url: str | None) -> str:
    """Scheme/trailing-slash-insensitive key for matching media URLs."""
    u = (url or "").strip().lower()
    u = re.sub(r"^https?://", "", u).split("?", 1)[0].rstrip("/")
    return u


_HEADING_TAGS = ("h1", "h2", "h3", "h4", "h5", "h6")


@overload
def _strip_play_button_glyphs(content_html: str) -> str: ...
@overload
def _strip_play_button_glyphs(content_html: None) -> None: ...
def _strip_play_button_glyphs(content_html: str | None) -> str | None:
    """Remove a facade's standalone play-button glyph.

    A "video facade" is a thumbnail plus a play triangle the publisher positions
    ON TOP of it with CSS (guitarworld ships
    ``<svg class="play-button" width="234.67">`` next to the thumbnail). The
    sanitizer drops that positioning, so the glyph stops being an overlay and
    becomes a block element in the flow — rendering as a huge logo above the
    video. Overlaid or not, it is decoration: the recovered embed has a real
    play button of its own.
    """
    if content_html is None or "play-button" not in content_html:
        return content_html
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(content_html, "html.parser")
    removed = False
    for svg in soup.find_all("svg"):
        classes = svg.get("class") or []
        if isinstance(classes, str):
            classes = classes.split()
        if any("play-button" in str(c) for c in classes):
            svg.decompose()
            removed = True
    return str(soup) if removed else content_html


def _place_recovered_embeds(content_html: str, items: list[tuple[str | None, str]] | list[tuple[str, str]]) -> str:
    """Insert recovered embeds where they belong, not just at the bottom.

    Four passes, in order: (1) replace a bare body link that points at the same
    media — so the player takes the place of the link the feed showed instead of
    the embed; (2) fill empty video-container divs (a class hinting
    youtube/video with no content left inside — the husk a stripped facade
    leaves behind, e.g. guitarworld's ``<div class="youtube-video">``); (3)
    fill empty ``<p></p>`` placeholders that follow a heading (the stripped
    embed slots, e.g. theobelisk's ``<h3>title</h3><p></p>``), in document
    order; (4) append whatever's left at the bottom."""
    if not items:
        return content_html
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(content_html or "", "html.parser")
    remaining = list(items)

    # Pass 1 — replace a matching bare link.
    for canonical, embed in list(remaining):
        if not canonical:
            continue
        target = None
        for a in soup.find_all("a"):
            href = str(a.get("href") or "")
            if canonical.startswith("yt:"):
                vid = canonical[3:]
                if youtube_duration_service.extract_video_id(href) == vid or f"/embed/{vid}" in href:
                    target = a
                    break
            elif _norm_media_link(href) and _norm_media_link(href) == _norm_media_link(canonical):
                target = a
                break
        if target is None:
            continue
        repl = BeautifulSoup(embed, "html.parser")
        parent = target.parent
        if parent is not None and parent.name == "p" and parent.get_text(strip=True) == target.get_text(strip=True):
            parent.replace_with(repl)  # link is the paragraph's sole content
        else:
            target.replace_with(repl)
        remaining.remove((canonical, embed))

    # Pass 2 — fill empty video-container divs (stripped-facade husks), in
    # document order, so the player lands where the page had it.
    if remaining:
        husks = []
        for d in soup.find_all("div"):
            blob = " ".join(d.get("class") or []).lower()
            if not any(h in blob for h in ("youtube", "video-embed", "video-container", "video-aspect")) and blob != "video":
                continue
            if d.get_text(strip=True) or d.find(["img", "iframe", "audio", "video"]):
                continue
            if any(d in h.descendants for h in husks):
                continue  # inner box of a husk already claimed
            husks.append(d)
        for d, (canonical, embed) in zip(husks, list(remaining), strict=False):
            d.replace_with(BeautifulSoup(embed, "html.parser"))
            remaining.remove((canonical, embed))

    # Pass 3 — fill empty <p> placeholders that follow a heading.
    if remaining:
        empties = []
        for p in soup.find_all("p"):
            if p.get_text(strip=True) or p.find(["img", "iframe", "audio", "video", "figure"]):
                continue
            prev = p.find_previous_sibling()
            if prev is not None and prev.name in _HEADING_TAGS:
                empties.append(p)
        for p, (canonical, embed) in zip(empties, list(remaining), strict=False):
            p.replace_with(BeautifulSoup(embed, "html.parser"))
            remaining.remove((canonical, embed))

    out = str(soup)
    # Pass 4 — append leftovers.
    if remaining:
        out += "".join(f'<p class="lectio-embed">{e}</p>' for _, e in remaining)
    return out


_HUSK_CLASS_RE = re.compile(r'class="[^"]*(?:youtube|video-embed|video-container|video-aspect)[^"]*"')


def _body_has_embed_husk(body: str) -> bool:
    """True when the body contains an empty video-container div (the husk a
    stripped click-to-load facade leaves behind). Cheap regex prefilter, then
    the same emptiness check the placement pass uses."""
    if not body or not _HUSK_CLASS_RE.search(body):
        return False
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(body, "html.parser")
    for d in soup.find_all("div"):
        blob = " ".join(d.get("class") or []).lower()
        if not any(h in blob for h in ("youtube", "video-embed", "video-container", "video-aspect")):
            continue
        if not d.get_text(strip=True) and not d.find(["img", "iframe", "audio", "video"]):
            return True
    return False


def _inject_recovered_source_embeds(content_html, entry):
    """Recover media embeds from the source page when the feed body has none.

    Entries ingested before Lectio stopped stripping ``<iframe>`` at feed-parse
    time lost their YouTube/Bandcamp/SoundCloud players, and — unlike WordPress
    embed blocks — leave no placeholder figure to refill, so the feed-side
    recovery can't help. When the stored body carries no embed and the entry has a
    source link, fetch the page once (cached, SSRF-guarded via the lead-image
    source-HTML cache) and re-attach any allowlisted players it has, placed in
    context (see ``_place_recovered_embeds``) — mirroring the Reader-view
    ``_reinject_readability_embeds`` recovery."""
    link = (getattr(entry, "link", "") or "").strip()
    if not link:
        return content_html
    body = content_html if isinstance(content_html, str) else ""
    if "<iframe" in body.lower():
        return content_html  # already has an embed — nothing to recover
    cached = lead_image_service.get_cached_source_html(link)
    if cached is None:
        # Don't block the render on a network GET — queue a background fetch and
        # leave the body unchanged; the embed fills in on a later open. Many pages
        # are already cached by the lead-image scraper, so this often hits.
        lead_image_service.queue_source_html_fetch(link)
        # Exception: when the body carries a stripped-embed husk (an empty
        # video-container div), a player almost certainly belongs there — the
        # cache is just cold (in-memory, cleared on restart). Wait briefly for
        # the queued fetch so the first open shows the video too; the husk
        # check keeps ordinary embed-less entries on the fast path.
        if _body_has_embed_husk(body) and lead_image_service.wait_for_source_html_fetch(link, timeout=2.5):
            cached = lead_image_service.get_cached_source_html(link)
        if cached is None:
            return content_html
    _base, raw_html = cached
    items = _extract_source_embed_iframes(raw_html, body)
    if not items:
        return content_html
    return _place_recovered_embeds(_strip_play_button_glyphs(body), items)


def _slice_to_content(raw_html: str, selectors: tuple[str, ...]) -> str | None:
    """Reduce a page to the nodes a plugin named as its article body.

    Returns a document rather than a fragment so the caller's extraction keeps
    working normally on it. None when nothing matched, so a site that changes
    its markup falls back to whole-page extraction instead of losing the body.
    """
    try:
        from bs4 import BeautifulSoup

        soup = BeautifulSoup(raw_html, "html.parser")
        parts = [str(node) for sel in selectors for node in soup.select(sel)]
    except Exception:  # noqa: BLE001
        return None
    if not parts:
        return None
    # <base> carries the original document's URL resolution into the slice, so a
    # relative src in a lifted node still resolves.
    return f"<html><head></head><body>{''.join(parts)}</body></html>"


def _source_article_body(entry) -> str | None:
    """The source page's article, readability-extracted, or None.

    For a feed that ships text with no pictures, the images are not merely
    missing — their *placement* is missing too. Reading the source article back
    recovers both: on a paizo blog post the feed body has 0 images while
    readability over the page yields 7, interleaved through the text where the
    author put them, which a gallery bolted onto the end cannot reproduce.

    Render-time on purpose, never stored. A feed entry's body is overwritten by
    the next refresh — which is why re-fetch refuses feed-provided entries — so
    substituting here is the only form of this that survives.
    """
    # Synchronous, unlike the gallery's prime-and-retry. This is the narrow case
    # fetch_source_html_now documents: the page is the only place the article's
    # pictures exist, so deferring to a later open shows a body that is missing
    # them now — the same reason _inject_tapas_episode_panels fetches inline.
    #
    # The async path was tried first and does not work here. A paizo blog page is
    # ~1MB, far more than the 0.8s wait allows, and the cache is per-process so
    # every restart empties it: the first open after a deploy got no images at
    # all, because the gallery fallback needs that same cache. Only the first
    # open of an entry pays the fetch; the cache serves the rest.
    # Declining skips the synchronous fetch below too: the gallery then primes the page itself, so a first open after a restart may
    # show the feed text before the panels.
    if site_content_plugins.declines_source_article(entry.link):
        return None
    cached = lead_image_service.fetch_source_html_now(entry.link)
    if not cached:
        return None
    _base, raw_html = cached
    # When the site names its content, slice to it first and let the normal
    # extraction run over the slice — that keeps every sanitizing, lazy-media
    # and image-sizing step, while giving readability a document that is only
    # the article. Readability cannot separate "Back to Blog", a tag row, a
    # sharing widget or a related-posts rail from the piece, because they are
    # the same stuff by its measure.
    selectors = site_content_plugins.content_selectors(entry.link)
    sliced = _slice_to_content(raw_html, selectors) if selectors else None
    try:
        if sliced:
            # The slice is ALREADY only the article, so it goes through the
            # whole-body path. Running readability over it re-ran the very
            # judgement the selector exists to replace: given a run of sibling
            # section wrappers it kept the single highest-scoring one, which
            # emptied two of three test posts outright.
            _title, article_html = extract_full_page_article(sliced, entry.link)
        else:
            _title, article_html = extract_readability_article(raw_html, entry.link)
    except Exception:  # noqa: BLE001 — fall back to the gallery, never fail the render
        LOGGER.debug("source-article extraction failed for %s", entry.link, exc_info=True)
        return None
    # Only worth substituting if it actually carries the pictures AND reads like
    # the article: a thin or image-less extraction is worse than the feed's own
    # text, which is complete prose in exactly this case.
    if "<img" not in article_html.lower():
        return None
    if len(re.sub(r"<[^>]+>", " ", article_html).split()) < 50:
        return None
    if not _reads_like_feed_text(article_html, entry):
        return None
    return article_html


def _reads_like_feed_text(article_html: str, entry) -> bool:
    """False when the extraction shares almost no vocabulary with the feed's own text.

    A client-rendered page (tinyview) has no article in its served HTML, so readability returns the site's framework — logo, search
    bar, loading skeleton — which passes the word and image counts and replaced a real post. The feed's text is the reference: an
    extraction of the right page contains most of its words, even when the feed only ships an excerpt. Skipped when the feed has too
    little text to judge by."""
    feed_text = entry_content.entry_body(entry)
    feed_words = set(re.findall(r"[a-z]{4,}", html.unescape(re.sub(r"<[^>]+>", " ", feed_text)).lower()))
    if len(feed_words) < 10:
        return True
    article_words = set(re.findall(r"[a-z]{4,}", html.unescape(re.sub(r"<[^>]+>", " ", article_html)).lower()))
    return len(feed_words & article_words) / len(feed_words) >= 0.3


def _inject_source_gallery(content_html, entry, lead_image_url):
    """Append the source article's images to an image-less feed body.

    Extracted from get_entry_detail; the caller gates this on the per-feed
    inject_source_images pref + a present entry.link. Collects the source page's
    images (deduped against the lead and existing body images), priming the source
    HTML in the background and filling on a later open if it isn't cached yet."""
    _exclude_imgs: set[str] = set()
    if lead_image_url:
        _exclude_imgs.add(lead_image_url)
    for _m in re.finditer(r'<img\b[^>]*\bsrc=["\']([^"\']+)["\']', content_html or "", re.IGNORECASE):
        _exclude_imgs.add(html.unescape(_m.group(1)))
    _gallery = lead_image_service.extract_source_gallery_urls(entry.link, exclude_urls=_exclude_imgs)
    if not _gallery:
        # Prime the source HTML in the background, wait briefly, then retry — so the
        # gallery fills on first open for fast sites, later otherwise.
        lead_image_service.queue_source_html_fetch(entry.link)
        lead_image_service.wait_for_source_html_fetch(entry.link, timeout=0.8)
        _gallery = lead_image_service.extract_source_gallery_urls(entry.link, exclude_urls=_exclude_imgs)
    if _gallery:
        content_html, _gallery = _fold_body_panel_into_gallery(content_html, entry.link, lead_image_url, _gallery)
        _figs = "".join(
            f'<figure><img src="{html.escape(u, quote=True)}" loading="lazy" referrerpolicy="no-referrer"></figure>' for u in _gallery
        )
        content_html = (content_html or "") + f'<div class="source-gallery">{_figs}</div>'
    return content_html


_PANEL_PARA_RE = re.compile(r"<p\b[^>]*>\s*((?:<img\b[^>]*>\s*(?:<br\s*/?>\s*)*)+)</p>", re.IGNORECASE)
_TILE_MIN_PANEL_WIDTH = 300
_TILE_MAX_PANEL_WIDTH = 1200


def _img_int_attr(tag: str, name: str) -> int | None:
    m = re.search(rf"""\s{name}\s*=\s*["']?(\d{{1,5}})(?!\d)""", tag, re.IGNORECASE)
    return int(m.group(1)) if m else None


def _is_tileable_panel(img: str) -> bool:
    """A discrete panel: declared width and height, panel-sized, at most 1.3 tall-to-wide, and not floated.

    Same rule as the Webtoons grid: taller slices are one strip cut into pieces and must stay stacked, and an image with no
    dimensions cannot be judged. A button, banner or note falls outside the size window."""
    w, h = _img_int_attr(img, "width"), _img_int_attr(img, "height")
    if not w or not h or not _TILE_MIN_PANEL_WIDTH <= w <= _TILE_MAX_PANEL_WIDTH or h * 10 > w * 13:
        return False
    return not re.search(r"float\s*:", img, re.IGNORECASE)


def _tile_inline_webcomic_panels(content_html: str) -> tuple[str, bool]:
    """Lay a run of two or more discrete panels, one per paragraph or all in one, in the 2-across ``.source-gallery`` grid.

    For webcomic feeds that ship every panel inline rather than leaving them to inject_source_images. A
    paragraph with any image that is not a tileable panel, or any text between paragraphs, ends the run and is kept verbatim, as
    is a run of fewer than two images. Returns ``(content_html, tiled)``."""
    out: list[str] = []
    run: list[tuple[str, list[str]]] = []
    pos = 0
    tiled = False

    def flush() -> None:
        nonlocal tiled
        imgs = [img for _, group in run for img in group]
        if len(imgs) >= 2:
            figs = "".join(f"<figure>{img}</figure>" for img in imgs)
            out.append(f'<div class="source-gallery">{figs}</div>')
            tiled = True
        else:
            out.extend(orig for orig, _ in run)
        run.clear()

    for m in _PANEL_PARA_RE.finditer(content_html):
        gap = content_html[pos : m.start()]
        if gap.strip():
            flush()
            out.append(gap)
        pos = m.end()
        imgs = _IMG_TAG_RE.findall(m.group(1))
        if all(_is_tileable_panel(img) for img in imgs):
            run.append((m.group(0), imgs))
        else:
            flush()
            out.append(m.group(0))
    flush()
    out.append(content_html[pos:])
    return "".join(out), tiled


def _fold_body_panel_into_gallery(content_html, entry_link, lead_image_url, gallery: list[str]) -> tuple[str, list[str]]:
    """Move the body's one image into the gallery when it is one of the source page's panels.

    A webcomic's first panel reaches the body on its own (_inject_webcomic_panel_into_bodyless_entry), and the gallery then holds
    the rest — so panel 1 rendered full width above a 2-across grid of its siblings (tinyview, 2026-09-28). Only runs when the
    gallery already has other panels, so a single-panel comic keeps its full-width image."""
    srcs = [html.unescape(m.group(1)) for m in re.finditer(r'<img\b[^>]*\bsrc=["\']([^"\']+)["\']', content_html or "", re.IGNORECASE)]
    if len(srcs) != 1:
        return content_html, gallery
    full = lead_image_service.extract_source_gallery_urls(entry_link, exclude_urls={lead_image_url} if lead_image_url else set())
    body_path = urlparse(srcs[0]).path
    if not any(urlparse(u).path == body_path for u in full):
        return content_html, gallery
    stripped, n = re.subn(r"<p\b[^>]*>\s*<img\b[^>]*>\s*</p>", "", content_html, count=1, flags=re.IGNORECASE)
    if not n:
        stripped = re.sub(r"<img\b[^>]*>", "", content_html, count=1, flags=re.IGNORECASE)
    return stripped, full


_CAPTION_IMG_TAG_RE = re.compile(r"<img\b[^>]*/?>", re.IGNORECASE | re.DOTALL)
_CAPTION_ATTR_RE = re.compile(r'\b(src|title)\s*=\s*(?:"([^"]*)"|\x27([^\x27]*)\x27|(\S+))', re.IGNORECASE)
_TRIVIAL_ALT_TEXTS = frozenset(
    {
        "responsive image",
        "image",
        "photo",
        "picture",
        "img",
        "thumbnail",
        "banner",
        "featured image",
        "previous",
        "next",
        "first",
        "last",
        "random",
        "prev",
        "newer",
        "older",
        # Social share-button / analytics-pixel alt text (AddToAny "Share", statcounter
        # "Web Analytics") — never a real photo caption.
        "share",
        "web analytics",
        "analytics",
    }
)
_DECORATIVE_CAP_WORDS = frozenset(
    {
        "banner",
        "header",
        "image",
        "cover",
        "featured",
        "photo",
        "thumbnail",
        "logo",
        "graphic",
        "artwork",
        "illustration",
    }
)


def _initial_image_caption(content_html, entry, lead_image_url):
    """Find an initial caption (img title/alt) before opener stripping intactness is lost.

    Extracted from get_entry_detail. Prefers a ``title=`` on the img whose src matches
    the lead image (high confidence); otherwise accepts a title from a different img
    only when no separate lead image exists. Persisted alt/title (from a prior source
    scrape) override the in-feed title. Returns
    ``(image_title_text, in_feed_title_is_lead_img, persisted_alt, persisted_title)``."""
    image_title_text: str | None = None
    in_feed_title_is_lead_img = False
    for _search_html in (content_html, entry.summary):
        if not isinstance(_search_html, str):
            continue
        for _tag_m in _CAPTION_IMG_TAG_RE.finditer(_search_html):
            _tag_attrs: dict[str, str] = {}
            for _am in _CAPTION_ATTR_RE.finditer(_tag_m.group(0)):
                _k = _am.group(1).lower()
                _v = html.unescape((_am.group(2) or _am.group(3) or _am.group(4) or "").strip())
                if _k and _v:
                    _tag_attrs[_k] = _v
            _title_val = re.sub(r"<[^>]+>", "", html.unescape(_tag_attrs.get("title", ""))).strip()
            if not _title_val:
                continue
            _src_val = _tag_attrs.get("src", "")
            _matches_lead = bool(
                lead_image_url
                and _src_val
                and (
                    _src_val in lead_image_url
                    or lead_image_url in _src_val
                    or _src_val.split("?")[0].rstrip("/") == lead_image_url.split("?")[0].rstrip("/")
                )
            )
            if _matches_lead:
                image_title_text = _title_val
                in_feed_title_is_lead_img = True
                break
            if image_title_text is None and not lead_image_url:
                # Low-confidence title from a different image — only when no separate
                # lead image is known (else it'd belong to an unrelated inline img).
                image_title_text = _title_val
        if in_feed_title_is_lead_img:
            break

    # Persisted alt/title (from a prior background source-page scrape) wins over a
    # low-confidence in-feed title.
    persisted_alt = lead_image_service.get_entry_image_alt(str(entry.feed_url), str(entry.id))
    persisted_title = lead_image_service.get_entry_image_title(str(entry.feed_url), str(entry.id))
    if persisted_alt or persisted_title:
        image_title_text = persisted_title or persisted_alt  # title preferred
    return image_title_text, in_feed_title_is_lead_img, persisted_alt, persisted_title


def _suppress_junk_caption(image_title_text, entry):
    """Drop captions that aren't real photo captions: trivial alt text, date-only
    strings, or text that merely restates the article title (incl. auto-generated
    banner captions). Extracted from get_entry_detail; returns the caption or None."""
    if not image_title_text:
        return image_title_text
    if image_title_text.lower() in _TRIVIAL_ALT_TEXTS:
        return None
    if _DATE_ONLY_CAP_RE.match(image_title_text):
        return None
    if entry.title:
        _norm_cap = " ".join(re.sub(r"<[^>]+>", "", image_title_text).split()).lower()
        _norm_etitle = " ".join(html.unescape(str(entry.title)).split()).lower()
        if _norm_etitle and (
            _norm_cap == _norm_etitle
            or (len(_norm_etitle) > 20 and _norm_etitle in _norm_cap)
            or (len(_norm_cap) >= 8 and _norm_etitle.endswith(_norm_cap))
        ):
            return None
        # Auto-generated banner captions restate the title plus a decorative word
        # and/or a date — strip those tokens; if the remainder is wholly in the title,
        # the caption adds nothing. Only when the caption actually looked banner-like.
        _cap_tokens = re.findall(r"[a-z0-9]+", _norm_cap)
        _title_tokens = set(re.findall(r"[a-z0-9]+", _norm_etitle))
        _looks_banner_like = any(t in _DECORATIVE_CAP_WORDS for t in _cap_tokens) or bool(
            re.search(r"\b\d{4}\b|\b\d{1,2}[/.\-]\d{1,2}\b", _norm_cap)
        )
        _core_tokens = [t for t in _cap_tokens if t not in _DECORATIVE_CAP_WORDS and not t.isdigit()]
        if _looks_banner_like and _core_tokens and all(t in _title_tokens for t in _core_tokens):
            return None
    return image_title_text


def _apply_caption_source_pref(image_title_text, disp, entry, content_html):
    """Apply the per-feed caption_source preference (which raw attr to show).

    Extracted from get_entry_detail. "auto" keeps the computed caption (then runs the
    auto-suppress heuristic); none/alt/title/both override it from the persisted alt/
    title. Returns the final caption or None."""
    _caption_source = str(disp.get("caption_source") or "auto")
    _feed_url = str(entry.feed_url)
    _entry_id = str(entry.id)
    if _caption_source == "none":
        return None
    if _caption_source == "alt":
        return lead_image_service.get_entry_image_alt(_feed_url, _entry_id)
    if _caption_source == "title":
        return lead_image_service.get_entry_image_title(_feed_url, _entry_id)
    if _caption_source == "both":
        _ct = lead_image_service.get_entry_image_title(_feed_url, _entry_id)
        _ca = lead_image_service.get_entry_image_alt(_feed_url, _entry_id)
        return f"{_ct} — {_ca}" if (_ct and _ca and _ct != _ca) else (_ct or _ca)
    # "auto": keep the computed caption, but run the auto-suppress heuristic.
    if not should_show_caption(
        image_title_text,
        entry_title=entry.title,
        content_html=content_html,
        pref=int(disp.get("show_image_caption", -1)),
    ):
        return None
    return image_title_text


def _skip_lead_comments(html: str) -> int:
    """Offset past leading whitespace and HTML comments (e.g. Ghost kg-card-begin).

    Scanning in Python rather than in _LEAD_IMG_OPENER_RE: as a regex this was
    `(?:<!--.*?-->\\s*)*`, which backtracks exponentially on feed HTML shaped
    like "<!--" + "--><!--" * N whose closing "-->" never arrives (~1s at N=22,
    doubling per repetition). Feed bodies are attacker-influenced. A str.find
    loop is linear and needs no regex trickery to stay that way.

    An unterminated comment stops the scan, matching the old regex, which
    matched zero comments when it could not find the closer.
    """
    i, n = 0, len(html)
    while True:
        while i < n and html[i].isspace():
            i += 1
        if not html.startswith("<!--", i):
            return i
        end = html.find("-->", i + 4)
        if end < 0:
            return i
        i = end + 3


# Matched with an explicit pos from _skip_lead_comments, so no leading "^".
_LEAD_IMG_OPENER_RE = re.compile(
    # skip blank paragraphs (e.g. Blogger <p>&nbsp;</p>). No leading \s* here:
    # it would overlap with (?:&nbsp;|\s)* on plain whitespace, and the two
    # adjacent whitespace-matching quantifiers are ambiguous enough on a run of
    # spaces to blow up backtracking (ReDoS) when the trailing </p> never
    # arrives — feed-supplied HTML is attacker-influenced here.
    r"(?:<p\b[^>]*>(?:&nbsp;|\s)*</p>\s*)*"
    r"(?:<(?:p|figure|div)\b[^>]*>\s*){0,3}"
    r"(?:<a\b[^>]*>\s*)?"
    r"(?:<div\b[^>]*>\s*)?"  # allow one extra wrapper div after <a> (e.g. Substack image2-inset)
    r"<img\b[^>]*/?>",
    re.IGNORECASE | re.DOTALL,
)

# A float the sanitizer preserved, in its normalized form. Matched against the
# opener markup — the float usually sits on the wrapping <a> or <div>, not the
# <img> — so an author-floated opener can be left in the flow instead of being
# hoisted into a full-width hero. See _strip_lead_image_opener.
_FLOAT_STYLE_RE = re.compile(r"float:\s*(?:left|right)", re.IGNORECASE)
_CLOSE_A_RE = re.compile(r"</a\s*>", re.IGNORECASE)
_TUMBLR_MEDIA_PREFIX_RE = re.compile(r"^(https://64\.media\.tumblr\.com/[^/]+/[^/]+)/", re.IGNORECASE)

_OPENER_SRC_RE = re.compile(r"<img\b[^>]*?\bsrc=[\"']([^\"']+)[\"']", re.IGNORECASE | re.DOTALL)
# WordPress and most CDNs spell a resize as a -WxH suffix on the basename.
_IMG_SIZE_SUFFIX_RE = re.compile(r"-\d{2,5}x\d{2,5}(?=\.[A-Za-z0-9]+$)")


_OPENER_DIM_RE = re.compile(r"\b(width|height)=[\"']?(\d{2,5})", re.IGNORECASE)
_SRCSET_WIDTH_RE = re.compile(r"\s(\d{2,5})w(?:,|\s|$)")
# Below this, an opener is a thumbnail standing in for something better.
_FULL_SIZE_IMAGE_MIN_PX = 400


def _opener_is_own_full_image(opener_html: str, lead_image_url: str | None) -> bool:
    """True when the opener <img> is a full-size picture of its own, not a stand-in.

    Decides whether hoisting the lead may DELETE the opener from the body. Two
    situations reach here looking alike:

      * the opener is a PLACEHOLDER for the lead — ComicControl's
        ``/comicsthumbs/foo.jpg`` beside the full ``/comics/foo.jpg``, or a bare
        thumbnail in a body that is otherwise a line of text. Stripping is an
        upgrade.
      * the opener IS the article — claycomix ships the full 800x2455 strip in
        the body while og:image is a 1996x1349 preview. Stripping deleted the
        comic and showed the preview in its place, so the strip appeared nowhere.

    Deleting content needs positive evidence, but the placeholder case has to
    keep working, so both tests must pass: a different picture from the lead
    (basename, ignoring ``-WxH`` resize suffixes), AND declared dimensions that
    say full-size. An opener that declares nothing stays strippable, which is
    the long-standing behaviour for thumbnail-wrapper bodies.
    """
    if not lead_image_url:
        return False
    m = _OPENER_SRC_RE.search(opener_html or "")
    if not m:
        return False

    def _key(url: str) -> str:
        name = urlparse(html.unescape(url)).path.rsplit("/", 1)[-1]
        return _IMG_SIZE_SUFFIX_RE.sub("", name).lower()

    src_key = _key(m.group(1))
    if not src_key or src_key == _key(lead_image_url):
        return False  # same picture, resized or relocated — a placeholder

    biggest = max([int(v) for _a, v in _OPENER_DIM_RE.findall(opener_html)] + [int(w) for w in _SRCSET_WIDTH_RE.findall(opener_html)] + [0])
    return biggest >= _FULL_SIZE_IMAGE_MIN_PX


def _ad_asset_hashes() -> set[str]:
    """Cached per user: which /starred-asset/<hash> images are ad creatives.

    Cached because it scans every archived asset, and the answer only changes when
    new articles are captured. Invalidated on process restart, which is often
    enough for a set that grows slowly.
    """
    with _ad_asset_hashes_lock:
        cached = _ad_asset_hashes_cache.get("v")
        if cached is not None:
            return cached
    try:
        hashes = starred_archive_service.asset_ad_hashes()
    except Exception:  # noqa: BLE001 — never block a render on this
        LOGGER.warning("ad-asset hash scan failed", exc_info=True)
        hashes = set()
    with _ad_asset_hashes_lock:
        _ad_asset_hashes_cache["v"] = hashes
    return hashes


def _strip_ad_images(html_text):
    """Remove ad creatives from an article body, and their emptied wrappers.

    Publishers upload house ads to the same media directory as everything else, so
    the signals are the filename ("…-hero-superbanner.gif", "…-content-banner.jpg")
    and IAB slot sizes (728x90 and friends). In an ARCHIVED article the images have
    been rewritten to /starred-asset/<hash> and carry no filename at all, so those
    are matched against the archive's own record of each asset's size and original
    URL — otherwise the reader would keep showing ads the pane had dropped.
    """
    if not html_text or "<img" not in html_text.lower():
        return html_text
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(html_text, "html.parser")
    ad_hashes: set[str] | None = None
    removed = 0
    for img in soup.find_all("img"):
        src = str(img.get("src") or "")
        if not src:
            continue
        is_ad = lead_image_service.is_ad_url(src)
        if not is_ad and STARRED_ASSET_URL_PREFIX in src:
            if ad_hashes is None:
                ad_hashes = _ad_asset_hashes()
            digest = src.rsplit("/", 1)[-1].split("?", 1)[0]
            is_ad = digest in ad_hashes
        if not is_ad:
            continue
        # Climb through wrappers the image leaves empty (a linked ad is
        # <a><img></a>, often inside a centering <div>), so no gap is left behind.
        target = img
        for _ in range(3):
            parent = target.parent
            if parent is None or parent.name in ("body", "[document]", None):
                break
            if parent.get_text(strip=True) or len(parent.find_all("img")) > 1:
                break
            target = parent
        target.decompose()
        removed += 1
    if not removed:
        return html_text
    return str(soup).strip() or None


_BLOCK_SPACER_TAGS = frozenset({"div", "p", "span", "i", "b", "em", "strong", "figure"})
# Real blocks only, for the "is this <br> at a block boundary" test. Inline tags
# are deliberately absent: a <br> before a <span> is separating text, and removing
# it would join two lines the author meant to keep apart.
_REAL_BLOCK_TAGS = frozenset(
    {
        "div",
        "p",
        "figure",
        "blockquote",
        "table",
        "ul",
        "ol",
        "li",
        "pre",
        "hr",
        "h1",
        "h2",
        "h3",
        "h4",
        "h5",
        "h6",
        # <img> is inline in the HTML spec but not in Lectio: `.entry-content img`
        # is `display: block` without exception, so a <br> next to one can never be
        # separating text — it is always padding the author added to push the next
        # block down. mahonoir ships `<img><br/><br/><div>…`, which rendered as a
        # large gap between a single-panel comic and its commentary while its
        # multi-panel posts (no <br>) sat correctly. The reasoning that keeps inline
        # tags out of this set is exactly what puts <img> in it.
        "img",
    }
)


def _collapse_block_spacers(content_html):
    """Drop blocks whose only content is <br>/whitespace, and <br> sitting at a
    block boundary. Returns the (possibly None) html.

    These are spacing hacks, not content: Blogger emits
    ``</div><br/><div><i><br/></i>`` between blocks, which renders as a large
    empty gap. It was always there — a second image above it used to fill the
    space, so removing that image is what made it visible.

    Deliberately narrow. A ``<br><br>`` *inside* a paragraph is how plenty of
    feeds separate lines when they ship no <p> tags at all, and collapsing those
    would reflow real prose. Only breaks adjacent to a block boundary, and blocks
    holding nothing but breaks, are touched.
    """
    if not content_html or "<br" not in content_html.lower():
        return content_html
    from bs4 import BeautifulSoup, NavigableString

    soup = BeautifulSoup(content_html, "html.parser")

    def _neighbour(node, forward: bool):
        # Walk outward past inline wrappers (span/i/b/strong/em/...) when the
        # current level runs out of siblings, so an element nested a few inline
        # tags deep -- e.g. bitmapbooks.com's
        # <strong>Name</strong><i><span><br/></span></i><span>text</span> --
        # is judged against the enclosing real block, not the edge of its
        # innermost wrapper. Only stop climbing at an actual block tag (or the
        # root), which is a genuine block boundary.
        current = node
        while True:
            sib = current.next_sibling if forward else current.previous_sibling
            while sib is not None and isinstance(sib, NavigableString) and not sib.strip():
                sib = sib.next_sibling if forward else sib.previous_sibling
            if sib is not None:
                return sib
            parent = current.parent
            if parent is None or getattr(parent, "name", None) in _REAL_BLOCK_TAGS:
                return None
            current = parent

    def _is_boundary(el) -> bool:
        # None = the edge of the enclosing block, so sitting first/last is a
        # boundary too. A neighbouring <br> counts as a boundary so a run at a
        # block edge clears out one pass at a time.
        prev, nxt = _neighbour(el, False), _neighbour(el, True)
        prev_edge = prev is None or getattr(prev, "name", None) in _REAL_BLOCK_TAGS
        next_edge = nxt is None or getattr(nxt, "name", None) in _REAL_BLOCK_TAGS or getattr(nxt, "name", None) == "br"
        return prev_edge and next_edge

    def _is_spacer_only(el) -> bool:
        if el.name not in _BLOCK_SPACER_TAGS:
            return False
        if el.get_text(strip=True):
            return False
        if el.find(["img", "svg", "iframe", "video", "audio", "table"]) is not None:
            return False
        if el.find("br") is None:
            return False
        # An empty inline wrapper (span/i/b/...) holding only a <br> can also sit
        # mid-paragraph, between two real chunks of text -- bitmapbooks.com's
        # system-name headings do this. Only treat it as spacing, not a line
        # break the author wanted kept, when it's actually at a block boundary.
        return _is_boundary(el)

    # Innermost first, so <div><i><br/></i></div> collapses all the way out
    # rather than leaving the now-empty wrapper behind.
    for _ in range(3):
        victims = [el for el in soup.find_all(list(_BLOCK_SPACER_TAGS)) if _is_spacer_only(el)]
        if not victims:
            break
        for el in victims:
            if not getattr(el, "decomposed", False):
                el.decompose()

    # A <br> whose previous/next meaningful sibling is a block element is padding
    # between blocks, not a line break inside text.
    for br in list(soup.find_all("br")) + list(soup.find_all("br")):
        if getattr(br, "decomposed", False):
            continue
        if _is_boundary(br):
            br.decompose()

    out = str(soup).strip()
    return out or None


_HAS_IMG_RE = re.compile(r"<img\b", re.IGNORECASE)


def _inject_webcomic_panel_into_bodyless_entry(
    content_html, entry, feed_url: str, lead_image_url, body_had_image: bool = False, show_lead_in_article: bool = True
):
    """Put the comic in the article when a webcomic feed ships a body without one.

    mahonoir.com's feed carries no image at all — only a "The post … appeared
    first on …" paragraph — while advertising a per-entry social card as its
    `<enclosure>`. That card is a good LIST THUMBNAIL and a poor article: reading
    the entry showed a share graphic and no comic. Unlike claycomix, nothing can
    be recovered from the body, because the body never had it.

    So the panel is fetched from the source page and placed in the article. The
    separate hero is dropped in the same move, or the card and the comic both
    render. The list thumbnail is unaffected: the caller captured the resolved
    lead in `_resolved_lead_for_cache` before this ran, and that is what reaches
    `entry_lead_images`.

    Narrow on purpose — webcomic feeds only, and only when the body has no image
    of its own — because it costs a source-page fetch on the render path. The
    fetch is served from `_source_html_cache` when the same page was already
    pulled this session.

    ``body_had_image`` (checked BEFORE `_strip_lead_image_opener` ran) tells
    the mahonoir case (never had a picture) apart from misfile.com's: its body
    opens with the comic already, `_strip_lead_image_opener` matches it against
    the resolved lead and removes it as a duplicate — the body looks identical
    to mahonoir's afterwards (no `<img>` either way) even though the article
    already has its comic, correctly shown as the hero. Without this flag every
    open of an entry like that re-fetched the source page for a panel that was
    already resolved, wastefully and in a race with the same-URL background
    fetch `_inject_recovered_source_embeds` queues earlier in get_entry_detail.

    ``body_had_image`` only means "already shown as the hero" when
    ``show_lead_in_article`` is also true. `_strip_lead_image_opener` strips a
    matching opener out of the body even when the per-feed "show lead image in
    article" preference is OFF -- that setting means "never show this image at
    the top," feed-supplied opener included, not "only skip Lectio's own copy."
    So a webcomic feed with that preference off strips the body's comic and
    never renders a separate hero either, leaving the article with no comic at
    all unless the panel is (still) fetched here.
    """
    if not feed_url or not lead_image_service._is_feed_webcomic(feed_url):
        return content_html, lead_image_url
    # A <video> is the post's own media too: a Bluesky clip's poster frame "injected" above it rendered the video twice (tinysnek).
    if content_html and (_HAS_IMG_RE.search(content_html) or re.search(r"<video\b", content_html, re.IGNORECASE)):
        return content_html, lead_image_url
    if body_had_image and show_lead_in_article:
        return content_html, lead_image_url
    link = str(getattr(entry, "link", "") or "")
    if not link.startswith(("http://", "https://")):
        return content_html, lead_image_url
    try:
        if lead_image_service._plugin_should_skip_source_lookup(entry_link=link):
            # A plugin owning this host has already said the page will mislead a
            # scan, and the resolved lead image IS its answer. Penny Arcade is
            # exactly that: the page's first <img> is panel 1, so scanning here
            # put a single panel in the article and threw away the full strip the
            # plugin had just resolved — reintroducing, on the render path, the
            # bug the plugin exists to prevent.
            panel = lead_image_url
        else:
            # mahonoir and friends: nothing usable was resolved (the feed's
            # enclosure is a share card), so the page really is the only source.
            panel = lead_image_service._fetch_source_lead_image(link, is_webcomic=True)
            # Free: reuses the page HTML the fetch just above cached. Catches a
            # newly-locked strip (cad-comic.com and the like) at render time
            # rather than waiting for the next backfill pass over this feed.
            _entry_id = getattr(entry, "id", None)
            if _entry_id:
                lead_image_service.check_and_cache_webcomic_lock(feed_url, str(_entry_id), link)
    except Exception:  # noqa: BLE001 — an article without its comic beats a 500
        LOGGER.warning("[webcomic] panel injection failed for %s", link, exc_info=True)
        return content_html, lead_image_url
    if not panel:
        return content_html, lead_image_url
    figure = f'<p><img src="{html.escape(panel, quote=True)}" alt="" /></p>'
    return figure + (content_html or ""), None


# Tapas serves two different things from the same CDN: `/sa/` is *series art* —
# one image per episode, thumbnail-grade, and what the RSS feed ships — while
# `/c/` is the episode's actual content, one URL per panel. An episode with four
# panels arrives in the feed as a single `/sa/` picture.
_TAPAS_CONTENT_IMG_RE = re.compile(r'https://[a-z0-9-]+\.tapas\.io/c/[^"\'\s<>\\]+', re.IGNORECASE)
# Matching an <img> *and* its src in one pattern needs two `[^"\']*` runs around
# the host, and on feed-supplied HTML CodeQL rightly called that polynomial: a
# tag carrying many repetitions of `//x.tapas.io/sa/` backtracks quadratically
# (py/polynomial-redos, alert 192, raised on this very branch). The tag scan is
# one linear `[^>]*` instead, and the host test runs in Python on the extracted
# src — which is also a more honest test than a regex spelling of a hostname.
_IMG_TAG_RE = re.compile(r"<img\b[^>]*>", re.IGNORECASE)
_IMG_SRC_RE = re.compile(r"""\bsrc\s*=\s*(?:"([^"]*)"|'([^']*)')""", re.IGNORECASE)


def _img_src(tag: str) -> str:
    m = _IMG_SRC_RE.search(tag)
    if not m:
        return ""
    return html.unescape((m.group(1) or m.group(2) or "").strip())


def _strip_imgs(content_html: str, is_target) -> str:
    """Drop every `<img>` whose src `is_target` accepts. Linear in the input."""
    return _IMG_TAG_RE.sub(
        lambda m: "" if is_target(_img_src(m.group(0))) else m.group(0),
        content_html,
    )


def _is_tapas_series_art(src: str) -> bool:
    """`/sa/` on the Tapas CDN — series art, i.e. the thumbnail, not the comic."""
    try:
        parsed = urlparse(src)
    except ValueError:
        return False
    host = (parsed.hostname or "").lower()
    return (host == "tapas.io" or host.endswith(".tapas.io")) and parsed.path.startswith("/sa/")


# A Tapas body is literally `<p>null</p><img src=…/>` when the author wrote no
# caption — the API's null serialized into the feed.
_TAPAS_NULL_PARA_RE = re.compile(r"<p>\s*null\s*</p>", re.IGNORECASE)
_TAPAS_MAX_PANELS = 60


def _panels_into_body(content_html, panels: list[str], is_stripped_img, wide: frozenset[str] = frozenset(), tiled: bool = False):
    """Replace a thumbnail-grade body with the real panels.

    Shared by the Tapas and Webtoons injections: strip the feed's own picture
    out of the body (it is the list thumbnail, not the article), drop the
    separate hero with it — otherwise the thumbnail renders directly above the
    comic it is a thumbnail of — and put the panels in front of whatever prose
    is left. Returns ``(content_html, lead_image_url)``.

    ``tiled`` lays the panels in the same 2-across ``.source-gallery`` grid the other
    webcomics use; ``wide`` names panels (banners) that span both columns.
    """
    body = _strip_imgs(content_html or "", is_stripped_img)
    body = _TAPAS_NULL_PARA_RE.sub("", body).strip()
    if tiled:
        figs = "".join(
            ('<figure class="source-gallery-wide">' if u in wide else "<figure>")
            + f'<img src="{html.escape(u, quote=True)}" alt="" loading="lazy" referrerpolicy="no-referrer"></figure>'
            for u in panels
        )
        return f'<div class="source-gallery">{figs}</div>' + body, None
    figures = "".join(f'<p><img src="{html.escape(u, quote=True)}" alt="" /></p>' for u in panels)
    return figures + body, None


# The episode's own slices carry class="_images"; their `data-url` is the real
# source (the `src` is a spinner placeholder). Bounding on the class matters —
# the page also embeds a recommendation strip of other series, and a looser scan
# swept 62 URLs into an episode that has 50.
_WEBTOONS_SLICE_RE = re.compile(r'<img\b[^>]*\bclass="[^"]*\b_images\b[^"]*"[^>]*>', re.IGNORECASE)
_WEBTOONS_DATA_URL_RE = re.compile(r'\bdata-url="([^"]+)"', re.IGNORECASE)
# The feed's own picture, plus Webtoons page chrome (`webtoons-static`, e.g. the
# `bg_transparency.png` spacer) — neither belongs in the article once the real
# slices are in. A host set rather than a regex, for the ReDoS reason above.
_WEBTOONS_IMG_HOSTS = frozenset(
    {
        "webtoon-phinf.pstatic.net",
        "swebtoon-phinf.pstatic.net",
        "webtoons-static.pstatic.net",
    }
)


def _is_webtoons_body_img(src: str) -> bool:
    try:
        return (urlparse(src).hostname or "").lower() in _WEBTOONS_IMG_HOSTS
    except ValueError:
        return False


_WEBTOONS_MAX_PANELS = 80
_WEBTOONS_DIM_RE = re.compile(r'\b(width|height)="([0-9.]+)"', re.IGNORECASE)
# An episode is tiled only when it is discrete panels (Table Top: ~800x970 each)
# rather than one tall strip cut into slices (up to 800x1280, contiguous — tiling
# those would break the reading order). Slices above this height/width ratio mean
# "strip"; below 0.5 is a banner, which spans both columns instead.
_WEBTOONS_PANEL_MAX_RATIO = 1.3
_WEBTOONS_BANNER_MAX_RATIO = 0.5
_WEBTOONS_MIN_TILED_PANELS = 2
# Some creators (Table Top) post every panel singly and then repeat them as
# exactly 800x1000 four-panel composites at the end. A trailing run of those,
# preceded by at least four singles per composite, is the same content twice:
# keep the composites and drop the singles.
_WEBTOONS_COMPOSITE_SIZE = (800.0, 1000.0)
_WEBTOONS_COMPOSITE_PANELS = 4


def _webtoons_public_slice_url(url: str) -> str:
    """The publicly fetchable spelling of an episode slice.

    The page serves slices from `webtoon-phinf`, which answers **403 to anything
    without a `webtoons.com` Referer** — including a browser loading the image
    from our page. The sibling `swebtoon-phinf` host serves the same path with no
    Referer at all (verified 200 on every slice of three different series), and
    it is the host the RSS feed itself uses. So we rewrite to it rather than
    forging a Referer we do not have: same bytes, nobody's hotlink policy
    circumvented, and no image-proxy change needed.
    """
    url = html.unescape(url).strip()
    url = url.split("?", 1)[0]  # ?type=q90 / ?type=optimize
    return url.replace("//webtoon-phinf.", "//swebtoon-phinf.")


def _inject_webtoons_episode_panels(content_html, entry, feed_url: str, lead_image_url):
    """Put the whole strip in a Webtoons article, not the one image the feed sends.

    A Webtoons episode is a vertical strip cut into slices — 50 of them on a
    Backchannel chapter, 8 on MercWorks, 5 on False Knees — and the RSS feed
    carries exactly one image. That single picture is a fine list thumbnail and
    a poor article. Same trade as the Tapas injection above, and the same
    guarantee: the list thumbnail is untouched because the caller captured the
    resolved lead in `_resolved_lead_for_cache` before this ran.
    """
    link = str(getattr(entry, "link", "") or "")
    try:
        host = (urlparse(link).hostname or "").lower()
    except ValueError:
        return content_html, lead_image_url
    if not (host == "webtoons.com" or host.endswith(".webtoons.com")):
        return content_html, lead_image_url
    try:
        fetched = lead_image_service.fetch_source_html_now(link)
    except Exception:  # noqa: BLE001 — the feed's image beats a 500
        LOGGER.warning("[webtoons] panel fetch failed for %s", link, exc_info=True)
        return content_html, lead_image_url
    if not fetched:
        return content_html, lead_image_url
    _base, page_html = fetched

    panels: list[str] = []
    ratios: dict[str, float] = {}
    sizes: dict[str, tuple[float, float]] = {}
    seen: set[str] = set()
    for tag in _WEBTOONS_SLICE_RE.findall(page_html):
        m = _WEBTOONS_DATA_URL_RE.search(tag)
        if not m:
            continue
        url = _webtoons_public_slice_url(m.group(1))
        if not url.startswith("https://") or url in seen:
            continue
        seen.add(url)
        panels.append(url)
        dims = {k.lower(): v for k, v in _WEBTOONS_DIM_RE.findall(tag)}
        try:
            sizes[url] = (float(dims["width"]), float(dims["height"]))
            ratios[url] = sizes[url][1] / sizes[url][0]
        except KeyError, ValueError, ZeroDivisionError:
            pass
        if len(panels) >= _WEBTOONS_MAX_PANELS:
            break
    if not panels:
        return content_html, lead_image_url
    # A single-slice episode whose one slice is already the feed's image (Sarah's
    # Scribbles) gains nothing from the rewrite — but it costs nothing either,
    # and the strip-then-inject leaves exactly the same picture in place.
    wide = frozenset(u for u in panels if ratios.get(u, 1.0) < _WEBTOONS_BANNER_MAX_RATIO)
    # Strip the repeated singles when the episode ends in four-panel composites.
    trail = 0
    for u in reversed(panels):
        if sizes.get(u) != _WEBTOONS_COMPOSITE_SIZE:
            break
        trail += 1
    singles = [u for u in panels[: len(panels) - trail] if u not in wide]
    if trail and len(singles) >= _WEBTOONS_COMPOSITE_PANELS * trail:
        keep = [u for u in panels if u in wide or u in panels[len(panels) - trail :]]
        # Composites are already a grid: stacked, not tiled.
        return _panels_into_body(content_html, keep, _is_webtoons_body_img, wide=wide, tiled=False)
    body_panels = [u for u in panels if u not in wide]
    tiled = (
        len(body_panels) >= _WEBTOONS_MIN_TILED_PANELS
        and all(u in ratios for u in panels)
        and all(ratios[u] <= _WEBTOONS_PANEL_MAX_RATIO for u in body_panels)
    )
    return _panels_into_body(content_html, panels, _is_webtoons_body_img, wide=wide, tiled=tiled)


def _inject_tapas_episode_panels(content_html, entry, feed_url: str, lead_image_url):
    """Put the actual comic in a Tapas article, not the series art.

    The feed's `<content:encoded>` is one `/sa/` image — the same asset the
    episode page uses as its thumbnail. It makes a good LIST THUMBNAIL and a
    poor article: a four-panel episode read as a single picture, and the reader
    reported it as "comic img is wrong, but good for thumb". The panels live on
    the page as `/c/` URLs, one per panel.

    The list thumbnail is unaffected — the caller captured the resolved lead in
    `_resolved_lead_for_cache` before this ran. The separate hero *is* dropped,
    or the thumbnail renders above the comic it is a thumbnail of.

    Costs a source-page fetch on the render path, so it is narrow: Tapas links
    only, and only when the body has no `/c/` image already. The `/c/` URLs are
    signed and short-lived, which is why the page is re-read rather than the
    URLs stored — and why `__token__` is stripped from the image proxy's cache
    key, so the *bytes* are cached once and outlive every token.
    """
    link = str(getattr(entry, "link", "") or "")
    try:
        host = urlparse(link).hostname or ""
    except ValueError:
        return content_html, lead_image_url
    host = host.lower()
    if not (host == "tapas.io" or host.endswith(".tapas.io")):
        return content_html, lead_image_url
    if content_html and _TAPAS_CONTENT_IMG_RE.search(content_html):
        return content_html, lead_image_url  # already has the real panels
    try:
        fetched = lead_image_service.fetch_source_html_now(link)
    except Exception:  # noqa: BLE001 — the series art beats a 500
        LOGGER.warning("[tapas] panel fetch failed for %s", link, exc_info=True)
        return content_html, lead_image_url
    if not fetched:
        return content_html, lead_image_url
    _base, page_html = fetched

    panels: list[str] = []
    seen: set[str] = set()
    for raw in _TAPAS_CONTENT_IMG_RE.findall(page_html):
        url = html.unescape(raw).rstrip("\\")
        # Same file, different token, is the same panel — compare without it.
        key = url.split("?", 1)[0]
        if key in seen:
            continue
        seen.add(key)
        panels.append(url)
        if len(panels) >= _TAPAS_MAX_PANELS:
            break
    if not panels:
        return content_html, lead_image_url

    return _panels_into_body(content_html, panels, _is_tapas_series_art)


_IMG_TAG_ANY_RE = re.compile(r"<img\b[^>]*>", re.IGNORECASE)


def _strip_lead_image_opener(content_html, lead_image_url, feed_url: str, show_lead_in_article: bool):
    """Dedup the lead image against the article body. Returns (content_html, lead_image_url).

    Extracted from get_entry_detail. When the lead image is shown at the top:
    - if the body opens with that image, strip the opener (BS4 to remove empty
      ancestor containers cleanly; regex fallback when the opener is a *different*
      image, e.g. a thumbnail placeholder vs the full-size lead) — UNLESS the
      opener is the post's entire content, in which case stripping it would leave
      an empty body: the opener stays in place instead, and the separate lead is
      suppressed the same way the "appears later in the body" case below is;
    - if the lead image appears later in the body, drop the separate lead (show it in
      place) — except artwork feeds, where it's hoisted to the top instead;
    - Tumblr size-variant dedup (same media hash, different size suffix);
    - when the lead came from source scraping and the body is just a thumbnail
      wrapper (minimal text), strip the inline imgs."""
    if not (lead_image_url and isinstance(content_html, str)):
        return content_html, lead_image_url

    if not show_lead_in_article:
        # "Don't show the lead image in the article" has to include the case
        # where the FEED put it there. This setting used to suppress only
        # Lectio's own copy above the article, so a feed whose body opens with
        # the same image still showed it — unchecking the box did nothing
        # visible, which is how it was reported.
        #
        # Blogger is the case in hand: it emits the social-preview asset
        # (…-Blog-Metadata.png) as a real <img> at the top of the post body, and
        # that is also what the lead-image scorer picks as the thumbnail. So the
        # article opened with a near-identical crop of its own first picture.
        # Good thumbnail, unwanted in the body — exactly the split this setting
        # is for.
        #
        # Only the *opener* is stripped, and only when it is the lead image
        # itself. An occurrence further down is the author placing it in the
        # flow, which is content rather than a header.
        if _LEAD_IMG_OPENER_RE.match(content_html, _skip_lead_comments(content_html)):
            _stripped = _bs4_strip_opener(content_html, lead_image_url)
            if _stripped is not None:
                content_html = _stripped or None
        return content_html, lead_image_url

    _m = _LEAD_IMG_OPENER_RE.match(content_html, _skip_lead_comments(content_html))
    if _m and _FLOAT_STYLE_RE.search(_m.group(0)):
        # The author FLOATED this image, so the text is written to wrap around
        # it. Hoisting it to a full-width hero above the article destroys exactly
        # the layout it was given — the post reads as a centred block with all
        # its text pushed below, which is how it was reported (2026-08-02, a
        # Blogger review whose cover sits right with the review beside it).
        #
        # This is the same rule the branch below already applies to an image
        # further down the body: an image the author placed in the flow is
        # content, not a header. A float is that placement stated explicitly, and
        # it happens to be at the top. So leave the body untouched and drop the
        # separate lead, or the picture appears twice.
        #
        # Only the ARTICLE lead is dropped. The list thumbnail is resolved
        # independently (get_cached_entry_thumbnail), so the post keeps its
        # thumbnail either way.
        return content_html, None

    if _m:
        # BS4 removes the opener <img> and its now-empty ancestor containers without
        # touching sibling figures or anchored link text ("New comic!").
        _bs4_stripped = _bs4_strip_opener(content_html, lead_image_url)
        if _bs4_stripped is not None:
            _stripped_content = _bs4_stripped or None
            if _stripped_content:
                _stripped_content = (
                    re.sub(
                        r"^(?:\s*(?:<p\b[^>]*>\s*(?:&nbsp;\s*)*</p>|<br\s*/?>\s*))+",
                        "",
                        _stripped_content,
                        flags=re.IGNORECASE,
                    ).strip()
                    or None
                )
            if _stripped_content:
                content_html = _stripped_content
            # else: stripping would leave the body empty -- the image WAS the
            # entire post (tamriel-rebuilt.org summary-only entries: one <img>,
            # no other text). Leave the opener in place instead of blanking the
            # body; the lead-in-content check below then drops the now-redundant
            # separate hero, so the post shows its one image instead of nothing.
        else:
            # lead_image_url isn't in the opener's <img> src. Two very different
            # situations look identical here, and stripping is only right for one:
            #
            #   * the opener is a small PLACEHOLDER for the lead (ComicControl's
            #     /comicsthumbs/foo.jpg beside the full /comics/foo.jpg). Strip
            #     it — the hero shows the better copy of the same picture.
            #   * the opener is a DIFFERENT, often better picture than the lead.
            #     claycomix puts the full 800x2455 strip in the body while
            #     og:image is a 1996x1349 preview; stripping deleted the comic
            #     from the article and showed the preview in its place, so the
            #     actual strip appeared nowhere at all.
            #
            # A placeholder shares its basename with the lead; a different
            # picture does not. When they differ, keep the author's image where
            # they put it and drop the separate hero — the same call the floated
            # -image branch above makes. The list thumbnail is resolved
            # independently (get_cached_entry_thumbnail), so it is unaffected.
            if _opener_is_own_full_image(_m.group(0), lead_image_url):
                return content_html, None
            # Raw-strip the opener; restore any anchored "New comic!" link text.
            _matched_opener = _m.group(0)
            content_html = content_html[_m.end() :].lstrip() or None
            _a_opener_m = re.search(r"<a\b[^>]*>", _matched_opener, re.IGNORECASE)
            if _a_opener_m and content_html:
                _close_m = _CLOSE_A_RE.search(content_html)
                if _close_m:
                    _between_text = re.sub(r"<[^>]+>", "", content_html[: _close_m.start()]).strip()
                    if _between_text:
                        content_html = _a_opener_m.group(0) + content_html
                    else:
                        content_html = content_html[_close_m.end() :].lstrip() or None
        # Another copy of the image left in the body means the hero would duplicate it. Only <img> tags count: an <a href> to the
        # full-size file survives the strip when the author's link spans the image AND the text after it (dorktower.com,
        # 2026-09-28), and counting it dropped the hero with the body's copy already gone — the comic showed nowhere.
        if (
            content_html
            and lead_image_url
            and any(lead_image_url in t or lead_image_url in html.unescape(t) for t in _IMG_TAG_ANY_RE.findall(content_html))
        ):
            lead_image_url = None
    elif lead_image_url and (lead_image_url in content_html or lead_image_url in html.unescape(content_html)):
        _entry_strategy, _, _ = lead_image_service.get_feed_strategy(feed_url)
        if _entry_strategy == "artwork":
            # Artwork mode (e.g. ArtStation): the image follows the description —
            # hoist it to the top by stripping it from its position in the content.
            _bs4_stripped = _bs4_strip_opener(content_html, lead_image_url)
            if _bs4_stripped is not None:
                content_html = _bs4_stripped or None
            else:
                lead_image_url = None
        elif bluesky.is_bsky_feed(feed_url):
            # Bluesky RSS carries no <img> in its real body — the only way this URL
            # can already be in content_html is get_entry_detail's own bsky-recovery
            # append (fetch_post_images), not the author placing it in the flow. That
            # append IS the post's real content, so leave it in the body and keep the
            # separate lead too, rather than treating it as author-placed duplication.
            # Except a video's poster: the <video> already shows that frame, so a hero of it is the clip twice.
            if any(lead_image_url in t for t in re.findall(r"<video\b[^>]*>", html.unescape(content_html), re.IGNORECASE)):
                lead_image_url = None
            # Likewise a link card's thumb: the card in the body already shows it.
            elif "data-bsky-card" in content_html:
                lead_image_url = None
        else:
            # Lead URL is buried mid-article (author placed it there) — show it in
            # its natural position, not as a separate top lead.
            lead_image_url = None

    # Tumblr CDN size-variant dedup: the cached lead (s1280x1920) and the in-content
    # photo (s640x960) share the {media_hash}/{token} prefix but differ by size, so
    # string equality above misses it.
    if show_lead_in_article and lead_image_url and isinstance(content_html, str):
        _tumblr_prefix_m = _TUMBLR_MEDIA_PREFIX_RE.match(lead_image_url)
        if _tumblr_prefix_m:
            _tumblr_prefix = _tumblr_prefix_m.group(1) + "/"
            if _tumblr_prefix in content_html or _tumblr_prefix in html.unescape(content_html):
                lead_image_url = None

    # Source-scraped lead + body that's essentially just a thumbnail wrapper (minimal
    # text) → strip the inline imgs so a small thumb doesn't sit below the full lead.
    if show_lead_in_article and lead_image_url and isinstance(content_html, str) and lead_image_url not in content_html:
        _remaining_imgs = len(re.findall(r"<img\b", content_html, re.IGNORECASE))
        if _remaining_imgs <= 1:
            # Tumblr guard: a remaining image with a different media prefix is a
            # genuine second photo — keep it.
            _skip_strip = False
            if _remaining_imgs == 1:
                _tumblr_lead_m = _TUMBLR_MEDIA_PREFIX_RE.match(lead_image_url)
                if _tumblr_lead_m:
                    _rem_src_m = re.search(r'<img\b[^>]*\bsrc=["\']([^"\']+)["\']', content_html, re.IGNORECASE)
                    if _rem_src_m:
                        _tumblr_rem_m = _TUMBLR_MEDIA_PREFIX_RE.match(html.unescape(_rem_src_m.group(1)))
                        if _tumblr_rem_m and _tumblr_rem_m.group(1) != _tumblr_lead_m.group(1):
                            _skip_strip = True
            if not _skip_strip:
                _no_imgs = re.sub(r"<img\b[^>]*/?>", "", content_html, flags=re.IGNORECASE)
                _text_only = html.unescape(re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", _no_imgs))).strip()
                if len(_text_only) < 120:
                    content_html = _no_imgs.strip() or None
    return content_html, lead_image_url


def _resolve_article_lead_image(entry, video_id, show_lead_in_article: bool):
    """Resolve the article's lead image and whether it's still pending.

    Extracted from get_entry_detail. Derives the lead image by feed strategy,
    discards avatar/headshot images, and — for a never-processed entry with no
    inline image — kicks off a background source-page fetch (waiting ≤0.8s so fast
    sites fill on first open). Returns ``(lead_image_url, pending)``; both are None/
    False when a YouTube embed was injected (the player replaces the lead image)."""
    lead_image_url = _derive_article_lead_image(entry)
    # Discard avatar/portrait images (author headshots, profile pics) that some
    # feeds embed as the first image; prefer no image over a face. Check path only —
    # CDN domains like "googleusercontent.com" contain "user" and would
    # false-positive on a full-URL search.
    if lead_image_url:
        _lead_parsed = urlparse(lead_image_url)
        if (
            lead_image_service._AVATAR_HINT_PATTERNS.search(_lead_parsed.path)
            or lead_image_service.is_ad_url(lead_image_url)
            or lead_image_service.is_social_icon_url(lead_image_url)
        ):
            lead_image_url = None
    # Try source scraping when the entry has never been processed (ABSENT cache) and
    # the feed provides no inline image — covers article-only feeds where the best
    # image lives on the source page. (A stored None is left alone: it conflates
    # auto-discovered "none" with a user-cleared image; the background job retries
    # negatives on its own schedule.)
    _cache_key = (str(entry.feed_url), str(entry.id))
    _cached_val = lead_image_service._cache.get(_cache_key, "ABSENT")
    _should_source_fetch = (
        _cached_val == "ABSENT" and lead_image_url is None and not lead_image_service._is_feed_none_strategy(str(entry.feed_url))
    )
    pending = False
    if _should_source_fetch and entry.link:
        # Queue the source-page fetch in the background, then wait briefly so a
        # fast-responding site fills the image on the very first open without a
        # polling round-trip. Cap at 0.8s — slow hosts (Squarespace, WordPress.com)
        # exceed it and fill in on the next open instead.
        lead_image_service.queue_source_fetch(str(entry.feed_url), str(entry.id), entry.link)
        _fetch_t0 = time.monotonic()
        lead_image_service.wait_for_source_fetch(str(entry.feed_url), str(entry.id), timeout=0.8)
        _fetch_ms = int((time.monotonic() - _fetch_t0) * 1000)
        LOGGER.info("[perf] entry_detail: source_fetch_wait=%dms %s", _fetch_ms, entry.link)
        lead_image_url = lead_image_service.extract_entry_thumbnail_url(entry, include_source_lookup=False)
        pending = (lead_image_url is None) and show_lead_in_article and not video_id
    # A YouTube embed already shows the video; a separate lead image (its thumbnail)
    # above the player would be redundant.
    if video_id:
        return None, False
    return lead_image_url, pending


def _apply_entry_media(content_html, entry, feed_url: str, entry_id: str):
    """Prepend the podcast audio player and append footer attachments to an entry.

    Extracted from get_entry_detail. Returns ``(content_html, audio_feed_suggestion)``:
    - injects an ``<audio>`` player (pointed at /entries/media/audio so expired signed
      URLs refresh server-side) when the entry has playable audio and none inline;
    - appends a download list for non-audio/non-image enclosures;
    - when there's no playable audio, surfaces a separate podcast-host feed to
      subscribe to (if discovered and not already subscribed)."""
    audio_feed_suggestion: str | None = None
    with get_meta_connection() as _mconn:
        audio_url = _resolve_entry_audio_url(_mconn, feed_url, entry_id, entry)
        if not audio_url:
            _sugg = _get_suggested_audio_feed(_mconn, feed_url)
            if _sugg and not _is_feed_subscribed(_mconn, _sugg):
                audio_feed_suggestion = _sugg
    if audio_url and (not isinstance(content_html, str) or "<audio" not in content_html.lower()):
        _safe_feed_url = quote_plus(feed_url)
        _safe_entry_id = quote_plus(entry_id)
        _media_play_url = f"/entries/media/audio?feed_url={_safe_feed_url}&entry_id={_safe_entry_id}"
        _media_dl_url = f"/entries/media/download?feed_url={_safe_feed_url}&entry_id={_safe_entry_id}"
        _track_title = html.escape(getattr(entry, "title", None) or "Audio", quote=True)
        # A trigger that loads the track into the persistent global player bar
        # (owned by static/media-player.js) instead of an inline <audio> element,
        # which would be ripped out of the DOM on pane-swap and stop playback.
        _audio_player = (
            f'<div class="podcast-player" data-audio-src="{_media_play_url}" '
            f'data-audio-download="{_media_dl_url}" data-audio-title="{_track_title}">'
            f'<button type="button" class="podcast-play-trigger" '
            f'aria-label="Play audio in player">'
            f'<span class="podcast-play-trigger-icon" aria-hidden="true">▶</span>'
            f'<span class="podcast-play-trigger-label">Play audio</span>'
            f"</button>"
            f'<a class="podcast-download-link" href="{_media_dl_url}" download>Download</a>'
            f"</div>"
        )
        content_html = _audio_player + (content_html or "")

    # Footer attachments — non-audio enclosures (magazine PDFs, EPUBs, etc.) that
    # never appear in the article body. The audio enclosure, if any, is already
    # shown as a player above, so it's excluded here.
    _attachments_html = _render_entry_attachments(entry, audio_url)
    if _attachments_html:
        content_html = (content_html or "") + _attachments_html
    return content_html, audio_feed_suggestion


def _visible_word_count(value: str | None) -> int:
    return entry_content.visible_word_count(value)


def _richest_content(entry, chosen):
    """See ``services.entry_content.richest_content``."""
    return entry_content.richest_content(entry, chosen)


def _resolve_entry_content_html(entry):
    """Resolve an entry's display HTML from its content/summary.

    Extracted from get_entry_detail: prefers HTML content, falls back to BBCode
    conversion (Nexus Mods), promotes bare-text/escaped-plaintext summaries
    (tracker.example) to real HTML, and repairs URL-encoded ``http%3A/`` schemes
    the reader library mangles into relative paths. Returns the HTML or None."""
    content = _richest_content(entry, entry.get_content(prefer_summary=False))
    content_html = None
    if content and content.value and content.is_html:
        content_html = content.value
    # BBCode fallback: feeds that emit BBCode markup instead of HTML
    # (e.g. Nexus Mods).  feedparser marks description as text/html but
    # the value has [b]...[/b] brackets.  Also handles is_html=False text.
    if content and content.value:
        _cv = content_html or content.value
        if _looks_like_bbcode(_cv):
            content_html = _bbcode_to_html(content_html if content_html else content.value)

    # Bare-text feeds (no HTML content): promote a URL/break-bearing plain-text
    # summary to content_html so links work and breaks render, instead of the
    # unstyled <pre> fallback. Returns None for genuinely plain prose, which
    # keeps the <pre> path. Runs before the <br>->paragraph pipeline below so a
    # promoted summary benefits from it too.
    if not content_html:
        content_html = _promote_plaintext_summary(getattr(entry, "summary", None))
    elif _looks_like_escaped_plaintext(content_html):
        # Some feeds (e.g. tracker.example news) declare their content as
        # text/html but actually ship escaped plain text — literal
        # ``&lt;br&gt;`` breaks, bare URLs, and double-escaped ``&amp;amp;``
        # ampersands — so it renders as inert escaped text. Promote it the
        # same way as a bare-text summary.
        content_html = _promote_plaintext_summary(content_html) or content_html

    # Bodies stored before the sanitizer rebuilt Instagram embeds still carry the fallback blockquote.
    if isinstance(content_html, str):
        content_html = html_sanitize.instagram_embeds(content_html)

    # Some feeds embed URL-encoded protocols in src attributes (e.g. http%3A// instead
    # of http://).  The reader library resolves these as relative paths, producing
    # URLs like https://example.com/path/http%3A/actual-host.com/image.png.
    # Recover the original URL by extracting and decoding the embedded scheme.
    if isinstance(content_html, str) and "%3A/" in content_html:
        content_html = re.sub(
            r'https?://[^"\'<\s]+/(https?)%3A/([^"\'<\s]+)',
            r"\1://\2",
            content_html,
            flags=re.IGNORECASE,
        )
    return content_html


_COMICSTHUMBS_IMG_SRC_RE = re.compile(r'(<img\b[^>]*?\bsrc=["\'])([^"\']*?/comicsthumbs/[^"\']+)(["\'])', re.IGNORECASE)
# ComicControl filenames carry a cache-bust unix-timestamp prefix, e.g.
# "1782426356-ARV1701_05.jpg". The stable part is everything after it.
_COMICCONTROL_TS_PREFIX_RE = re.compile(r"^\d{6,}-")


def _comiccontrol_stable_name(url: str) -> str:
    """Return a ComicControl image's filename with its cache-bust timestamp
    prefix removed (e.g. '…/1782426356-ARV1701_05.jpg' -> 'ARV1701_05.jpg')."""
    fname = url.rsplit("/", 1)[-1].split("?", 1)[0]
    return _COMICCONTROL_TS_PREFIX_RE.sub("", fname)


def _promote_comicsthumbs_in_content(content_html: str, full_lead_url: str | None) -> str:
    """Promote inline ComicControl /comicsthumbs/ images to the full panel.

    ComicControl gives the thumbnail and the full panel DIFFERENT cache-bust
    timestamp prefixes (e.g. comicsthumbs/1782426356-X.jpg vs comics/1782426355-X.jpg),
    so a naive /comicsthumbs/ -> /comics/ swap keeps the thumb's timestamp — and
    ComicControl answers that nonexistent timestamp with a 200 placeholder, not the
    image, breaking the comic. When the resolved lead image (the real /comics/<ts>-<file>
    read from the page) shares the same timestamp-stripped filename, substitute it
    directly.

    Otherwise the image is left exactly as the feed served it. The naive swap used
    to run as a fallback, which meant the *first* view of a new strip — before the
    lead image has been derived and cached — replaced a working 31KB thumbnail
    with a 11KB placeholder, so the comic appeared broken until a reload
    (atomic-robo, reported 2026-07-26). A small correct comic beats a big broken
    one, and the promotion still happens on every later view."""
    lead_name = _comiccontrol_stable_name(full_lead_url) if full_lead_url and "/comics/" in full_lead_url else None

    def _sub(m: re.Match) -> str:
        src = m.group(2)
        if lead_name and _comiccontrol_stable_name(src).lower() == lead_name.lower():
            return f"{m.group(1)}{full_lead_url}{m.group(3)}"
        return m.group(0)  # unverifiable — keep the feed's own thumbnail

    return _COMICSTHUMBS_IMG_SRC_RE.sub(_sub, content_html)


def _strip_trailing_recirculation_rail(content_html: str) -> str:
    """Drop a Future plc "more from us" block from the end of a feed body.

    pcgamer (and its sister sites) close content:encoded with a
    ``<div class="product">`` holding a full-width promo image and a rail of
    house links — "2026 games: All the upcoming games", "Best PC games: Our
    all-time favorites"… It is the site's recirculation widget, not the article,
    and the image is large enough to read as the post's own.

    Matched by SHAPE rather than by class alone: "product" is a generic name, and
    a div called that in the middle of a review is likely to be the thing being
    reviewed. Required: it is the LAST element, and it carries the vanilla image
    figure or at least three bolded house links.
    """
    if not content_html or "product" not in content_html:
        return content_html
    try:
        from bs4 import BeautifulSoup
        from bs4 import Tag as _Bs4Tag

        soup = BeautifulSoup(content_html, "html.parser")
        root = soup.body or soup
        blocks = [n for n in root.find_all("div") if "product" in (n.get("class") or [])]
        if not blocks:
            return content_html
        changed = False
        for block in blocks:
            # Last element of its parent. "Last" ignores whitespace AND empty
            # containers: pcgamer closes the article with a bare <div></div>
            # after the rail, which counted as a following element and left the
            # rail on four posts out of five.
            tail = []
            for sib in block.next_siblings:
                if not isinstance(sib, _Bs4Tag):
                    if str(sib).strip():
                        tail.append(sib)
                    continue
                if sib.get_text(strip=True) or sib.find("img"):
                    tail.append(sib)
            if tail:
                continue
            rail_links = [a for a in block.find_all("a") if a.find("strong")]
            if block.select_one("figure.van-image-figure") or len(rail_links) >= 3:
                block.decompose()
                changed = True
        if not changed:
            return content_html
        return (root.decode_contents() if root is not soup else str(soup)).strip()
    except Exception:  # noqa: BLE001 — a cleanup must never lose the article
        LOGGER.debug("recirculation-rail strip failed", exc_info=True)
        return content_html


def _apply_feed_content_cleanups(content_html, feed_url: str, entry_id: str):
    """Apply the per-site / generic feed-content cleanups to an entry's HTML.

    Extracted from get_entry_detail; operates on content_html only (returning the
    cleaned value) so the render path reads as a sequence of named stages. Covers:
    NASA leading-nav strip, mynorthwest "RELATED STORIES" block, Ghost kg-audio
    cards, the WordPress "appeared first on" footer, qwantz nav tables, sanitized
    embed-container iframes, and recovery of stripped YouTube embeds."""
    content_html = _strip_trailing_recirculation_rail(content_html) if isinstance(content_html, str) else content_html

    # IsThereAnyDeal waitlist: one compact line per game offer instead of a column of one-word lines.
    if isinstance(content_html, str) and itad_waitlist.is_itad_feed(feed_url):
        content_html = itad_waitlist.reformat(content_html)

    # NASA Science RSS (earthobservatory.nasa.gov) injects the full site secondary-navigation
    # into content:encoded before the article body. Strip any leading wp-block-nasa-blocks-*
    # divs by tracking div nesting depth so the article starts at actual content.
    if isinstance(content_html, str) and re.search(r"<div[^>]*\bwp-block-nasa-blocks-", content_html[:300], re.IGNORECASE):
        _stripped = content_html
        while True:
            _nm = re.match(r"\s*<div[^>]*\bwp-block-nasa-blocks-\w", _stripped, re.IGNORECASE)
            if not _nm:
                break
            _depth = 0
            _strip_end = 0
            for _dm in re.finditer(r"<(/?)div\b[^>]*>", _stripped, re.IGNORECASE):
                if _dm.group(1):
                    _depth -= 1
                    if _depth == 0:
                        _strip_end = _dm.end()
                        break
                else:
                    _depth += 1
            if _strip_end > 0:
                _stripped = _stripped[_strip_end:].lstrip()
            else:
                break
        content_html = _stripped or None

    # Rendered video-player chrome captured from live pages (extension saves):
    # JWPlayer serializes its entire control DOM (jw-* classes, jwp-carousel
    # widgets, Future plc's vid-present wrapper on loudersound/guitarworld/…).
    # Without the player scripts it renders as junk text ("0 seconds of …",
    # "Latest Videos From …" carousels) that the live page's own CSS hides —
    # strip the whole containers at render, which also cleans already-stored
    # captures.
    if isinstance(content_html, str) and ("jw-" in content_html or "jwp-" in content_html or "vid-present" in content_html):
        for _cls in ("vid-present", "jwplayer", "jw-wrapper", "jwp-carousel"):
            content_html = _strip_div_blocks_by_class(content_html, _cls)

    # Share widgets and never-filled lazy-load placeholders — the same "chrome
    # the site's own JS would have handled" problem, generically. Textless and
    # image-free only, so real related-post blocks and galleries can't match.
    if isinstance(content_html, str):
        content_html = _strip_js_dependent_chrome(content_html)

    # MyNorthwest injects a "RELATED STORIES" sidebar block (div.related.alignright)
    # after the first paragraph. It contains external article thumbnails that
    # confuse lead-image extraction and clutter the reading view.
    if isinstance(content_html, str) and "mynorthwest.com" in feed_url and "related" in content_html:
        content_html = _strip_div_blocks_by_class(content_html, "related", "alignright")

    # Ghost CMS embeds a JS-powered audio card that renders as a broken custom player
    # without its scripts. Replace the entire kg-audio-card widget with a native
    # <audio controls> element so it works in the reader.
    if isinstance(content_html, str) and "kg-audio-card" in content_html:
        content_html = _transform_kg_audio_cards(content_html)

    # Tidy the WordPress "The post … appeared first on …" footer: keep one,
    # drop plugin duplicates, and clean the double-encoded literal "<p>" tags
    # some feeds emit. Generic across feeds.
    if isinstance(content_html, str):
        content_html = _fix_wp_post_footer(content_html)

    # Dinosaur Comics (qwantz): strip the nav tables wrapping the comic and
    # keep only the comic image + dated author commentary.
    if isinstance(content_html, str) and "qwantz.com" in feed_url:
        content_html = _clean_qwantz_content(content_html)

    # Some feeds (e.g. Introversion Blog via feedburner) sanitize <iframe> tags by
    # replacing them with the literal text "<strong>iframe</strong>" inside a
    # class="embed-container" div, leaving an adjacent plain-text YouTube link.
    # Convert these pairs into a proper YouTube embed.
    if isinstance(content_html, str) and "embed-container" in content_html and "strong" in content_html:

        def _replace_bad_iframe(m: re.Match) -> str:
            raw_url = m.group(1)
            vid = youtube_duration_service.extract_video_id(raw_url)
            if not vid:
                return ""
            return _youtube_embed_html(vid)

        content_html = re.sub(
            r'<div[^>]*class=["\']embed-container["\'][^>]*>\s*<strong>iframe</strong>\s*</div>'
            r'\s*<a[^>]+href=["\']'
            r'(https?://(?:www\.)?(?:youtube\.com/watch\?[^"\'<\s]+|youtu\.be/[^"\'<\s]+))'
            r'["\'][^>]*>[^<]*</a>',
            _replace_bad_iframe,
            content_html,
            flags=re.IGNORECASE,
        )

    # Recover stripped YouTube embeds — feedparser removes the <iframe> from
    # WordPress YouTube blocks, leaving an empty figure with the video id
    # gone. Re-parse of the raw feed (sanitize off) caches the ids; inject
    # the player when cached, else queue a scan so it fills in next open.
    # Only needed for entries stored *before* feed ingest stopped stripping
    # iframes (services.reader_sanitize) — newer entries keep the real embed,
    # so skip recovery when an <iframe> is already present.
    if isinstance(content_html, str) and _YT_EMBED_PLACEHOLDER_RE.search(content_html) and "<iframe" not in content_html.lower():
        with get_meta_connection() as _vconn:
            _vids = _lookup_media_video(_vconn, feed_url, entry_id)
        if _vids:
            content_html = _inject_recovered_youtube_embeds(content_html, _vids)
        elif _vids is None:
            # Not scanned yet — force a scan (bypasses the audio TTL so a new
            # post's embed isn't held back by a long no-audio backoff).
            _queue_media_audio_scan(feed_url)

    # ComicControl webcomics (atomic-robo, everblue, …) embed only the small
    # /comicsthumbs/ image in feed content; the full-resolution panel is the
    # same filename under /comics/. Promote inline so the reader shows the
    # readable comic, not the thumbnail.
    if isinstance(content_html, str) and "comicsthumbs" in content_html.lower():
        _full_lead = lead_image_service.get_cached_lead_image_url(feed_url, entry_id)
        content_html = _promote_comicsthumbs_in_content(content_html, _full_lead)

    # Standalone bare YouTube links (own paragraph / lone anchor) → inline player.
    # Covers feeds where the oEmbed iframe was stripped and only the link remains.
    if isinstance(content_html, str):
        content_html = _embed_standalone_youtube_links(content_html)

    # Standalone bare Bandcamp album/track links → embed. Cache-first: resolves
    # immediately when the album page HTML is already cached; otherwise queues a
    # background fetch so the embed appears on the next open.
    if isinstance(content_html, str):
        content_html = _embed_standalone_bandcamp_links(content_html)

    # Apply the per-user YouTube embed-host preference to feed-native players
    # (the recovered/injected player above already uses youtube_embed_host()).
    if isinstance(content_html, str):
        content_html = _apply_youtube_embed_host(content_html)

    return content_html


def get_entry_detail(feed_url: str, entry_id: str) -> dict | None:
    _t0 = time.monotonic()
    with get_reader() as reader:
        entry = reader.get_entry((feed_url, entry_id), None)
        if not entry:
            return _build_orphan_entry_detail(feed_url, entry_id)
        _reader_get_ms = int((time.monotonic() - _t0) * 1000)
        if _reader_get_ms > 200:
            LOGGER.info("[perf] entry_detail: reader.get_entry=%dms %s", _reader_get_ms, entry_id)

        published_dt = entry_effective_date(entry)
        author_name = (getattr(entry, "authors_str", None) or "").strip() or None
        # DA Watch feed: byline gets an Unwatch ✕. The feed file only carries the newest 50 posts, so
        # posts that aged out before author capture shipped never got an author in reader — fall back
        # to our DA store (matched on link).
        da_watch_artist = None
        if deviantart_service.deviantart_feed_id_from_url(str(feed_url)) and str(feed_url) == deviantart_watch_feed_url():
            da_watch_artist = deviantart_service.username_from_url(str(entry.link or ""))
            if not author_name and entry.link:
                with get_meta_connection() as _da_conn:
                    _row = _da_conn.execute(
                        "SELECT author FROM deviantart_entries WHERE entry_url = ? AND author != '' LIMIT 1", (str(entry.link),)
                    ).fetchone()
                author_name = (str(_row["author"]) if _row else None) or da_watch_artist

        content_html = _resolve_entry_content_html(entry)

        # Bluesky RSS is text-only; append the post's images (recovered from the
        # AT Protocol API, incl. content-labeled posts) so the article shows them.
        # The list thumbnail is handled separately in extract_entry_thumbnail_url.
        if bluesky.is_bsky_feed(str(entry.feed_url)):
            _existing = content_html or ""
            if _existing and not re.search(r"</?[a-zA-Z!]", _existing):
                # The RSS body is plain text with real newlines; HTML collapses them to one line.
                _existing = _promote_plaintext_summary(_existing) or html.escape(_existing).replace("\n", "<br>")
                content_html = _existing
            _quote = bluesky.fetch_post_quote(str(entry.id))
            _quote_html = (
                f'<p><em>Quoting <a href="{html.escape(_quote["url"], quote=True)}" target="_blank"'
                f' rel="noopener noreferrer">@{html.escape(_quote["handle"])}</a></em></p>'
                if _quote
                else ""
            )
            _quote_after = bool(_quote and _quote["own_media"])
            if _quote_html and not _quote_after:
                # Label what follows (the quoted post's media); the quoting post's own media goes before it instead.
                _existing += _quote_html
                content_html = _existing
            _bsky_video = bluesky.fetch_post_video(str(entry.id))
            if _bsky_video:
                # A real <video> instead of a static thumb: data-bsky-hls-src is
                # picked up by app.js's initBskyVideoPlayers, which attaches
                # native HLS (Safari) or lazy-loads vendored hls.js for everyone
                # else. poster keeps today's thumbnail as the pre-play frame.
                #
                # width/height (when the embed carried an aspectRatio) let
                # applyPortraitImageCap cap a portrait video the same way it
                # already caps a portrait image -- without them, a vertical
                # phone-shot clip stretched to the article's full column width
                # via width:100%/height:auto renders extremely tall. The video
                # itself can't report this on its own before playback starts
                # (preload="none" means no videoWidth/videoHeight yet).
                _dims = (
                    f' width="{html.escape(_bsky_video["width"], quote=True)}" height="{html.escape(_bsky_video["height"], quote=True)}"'
                    if _bsky_video.get("width") and _bsky_video.get("height")
                    else ""
                )
                content_html = _existing + (
                    f'<p><video controls preload="none" playsinline{_dims}'
                    f' poster="{html.escape(_bsky_video["thumbnail"], quote=True)}"'
                    f' data-bsky-hls-src="{html.escape(_bsky_video["playlist"], quote=True)}"'
                    f' style="max-width:100%;"></video></p>'
                )
            else:
                _bsky_imgs = bluesky.fetch_post_images(str(entry.id))
                if _bsky_imgs:
                    _add = "".join(
                        f'<p><img src="{html.escape(u, quote=True)}" loading="lazy"'
                        f' referrerpolicy="no-referrer" style="max-width:100%;height:auto;"></p>'
                        for u in _bsky_imgs
                        if u not in _existing
                    )
                    content_html = _existing + _add
                else:
                    _card = bluesky.fetch_post_card(str(entry.id))
                    if _card:
                        _thumb = (
                            f'<img src="{html.escape(_card["thumb"], quote=True)}" loading="lazy"'
                            f' referrerpolicy="no-referrer" data-bsky-card="1" style="max-width:100%;height:auto;"><br>'
                            if _card["thumb"]
                            else ""
                        )
                        content_html = _existing + (
                            f'<p><a href="{html.escape(_card["uri"], quote=True)}" target="_blank" rel="noopener noreferrer">'
                            f"{_thumb}{html.escape(_card['title'] or _card['uri'])}</a></p>"
                        )
            if _quote_after:
                content_html = (content_html or "") + _quote_html

        # Has this body been hand-cleaned in the pane? Gates "Revert cleanup" in
        # the UI, and suppresses the embed recovery below — re-adding an embed
        # the user just deleted is the one way a cleanup could look undone.
        try:
            with get_meta_connection() as _edit_conn:
                content_edited = bool(
                    _edit_conn.execute(
                        "SELECT 1 FROM entry_content_edits WHERE feed_url = ? AND entry_id = ?",
                        (feed_url, entry_id),
                    ).fetchone()
                )
        except sqlite3.OperationalError:
            content_edited = False  # tenant DB predates the table

        # Per-site / generic feed-content cleanups (NASA nav, mynorthwest related
        # block, Ghost audio cards, WordPress footer, qwantz nav, embed-container
        # iframes, recovered YouTube embeds). Operates on content_html only.
        content_html = _apply_feed_content_cleanups(content_html, feed_url, entry_id)

        # Recover media embeds (YouTube/Bandcamp/SoundCloud/…) from the source
        # page for older entries whose <iframe> was stripped at ingest and left no
        # placeholder to refill. Skips when the body already has an embed; fetch is
        # cached and SSRF-guarded. Runs before the YouTube-feed injection below so
        # that path still wins for native YouTube feeds.
        if not content_edited and not (isinstance(feed_url, str) and feed_url.startswith("https://www.youtube.com/feeds/videos.xml?")):
            content_html = _inject_recovered_source_embeds(content_html, entry)

        # Bandcamp single-track esig players are domain-locked to the publisher and
        # show "not available" in Lectio; fall back to the album player. Covers
        # both feed-native and source-recovered embeds.
        content_html = _strip_bandcamp_track_signature(content_html)

        # --- YouTube embed injection ---
        # Only for YouTube feeds (feeds/videos.xml?channel_id=...)
        duration_seconds = None
        duration_display = None
        video_id = None
        if feed_url.startswith("https://www.youtube.com/feeds/videos.xml?") and entry.link:
            video_id = youtube_duration_service.extract_video_id(entry.link)
            if video_id:
                premiere_prefix = _youtube_premiere_prefix(video_id)
                if premiere_prefix:
                    duration_display = premiere_prefix
                else:
                    duration_seconds, duration_display = youtube_duration_service.get_cached_duration(video_id)

            # Determine base HTML to attach embed to (use existing content_html or fallback to summary)
            base_html = content_html if isinstance(content_html, str) and content_html.strip() else (entry.summary or "")

            # Only inject if not already present
            if video_id and ("/embed/" not in str(base_html)):
                embed_html = _youtube_embed_html(video_id)
                # Ensure base_html is wrapped as HTML
                if not isinstance(base_html, str):
                    base_html = ""
                # If base_html is plain text (no HTML tags), linkify bare URLs
                # so they render as clickable links rather than plain text.
                #
                # "Plain text" here still arrives HTML-ESCAPED — a YouTube
                # description ships "Sources &amp; further reading" — so escaping
                # it again rendered a literal "&amp;" on screen. Unescape first,
                # exactly as _promote_plaintext_summary does, then escape once.
                if base_html and not re.search(r"<[a-z]", base_html, re.IGNORECASE):
                    base_html = _linkify_plaintext_description(base_html)
                content_html = embed_html + f"<div>{base_html}</div>"

        # Podcast audio player + footer attachments + (when no audio) a suggestion
        # to subscribe to a separate podcast-host feed.
        content_html, _audio_feed_suggestion = _apply_entry_media(content_html, entry, feed_url, entry_id)

        manual_tags = get_manual_tags_for_resource(reader, entry.resource_id)
        raw_feed_tags = get_feed_tag_suggestions(entry.feed_url, entry.id)
        # Source-page fallback: entries whose feed never delivered <category>
        # data (aged out of the feed window, or a tag-stripping publisher) can
        # usually be tagged from the article page's article:tag/keywords metas.
        # Uses the lead-image service's source-HTML cache when primed (zero
        # extra requests); otherwise queues its background fetch so the tags
        # appear on the next open of this entry — same deferral as captions.
        if not raw_feed_tags and entry.link and url_guard.is_safe_outbound_url(str(entry.link)):
            try:
                _cached_page = lead_image_service.get_cached_source_html(str(entry.link))
                if _cached_page is not None:
                    _page_tags = feed_tags_service_mod.extract_page_tags(_cached_page[1], str(entry.link))
                    if _page_tags:
                        feed_tag_service.record_entry_tags(str(entry.feed_url), [(str(entry.id), _page_tags)], source="open")
                        # Re-derive through the dismissal-aware getter rather
                        # than using _page_tags directly: dismissing every
                        # suggested tag for a feed makes get_feed_tag_suggestions
                        # return [] just like "never harvested" does, so without
                        # this a freshly re-harvested (but dismissed) tag came
                        # back as a chip on every single open.
                        raw_feed_tags = get_feed_tag_suggestions(str(entry.feed_url), str(entry.id))
                else:
                    # feed_url/entry_id let the fetch worker persist harvested
                    # tags immediately (survives restarts/cache eviction);
                    # chips then appear on the next open of this entry.
                    lead_image_service.queue_source_html_fetch(
                        str(entry.link),
                        feed_url=str(entry.feed_url),
                        entry_id=str(entry.id),
                    )
            except Exception:
                LOGGER.debug("page-tag fallback failed for %s", entry.link, exc_info=True)
        # Tags are stored raw; normalize to Lectio tag format for display. The
        # chips are feed-filter controls ([- tag +] toggling the feed's
        # tag_filter rule), so all of the entry's feed tags show — including
        # ones already applied as manual tags.
        # The user's pinned tags first, exactly as /entries/feed-tags does it.
        # Both paths must agree: this one renders with the page, the other
        # injects after a tag is applied and the pane re-renders — and when only
        # one of them knew about pinned tags, applying one suggestion made the
        # rest disappear.
        entry_pinned_tags = get_feed_pinned_tags(str(entry.feed_url))
        _publisher_tags = {n for n in (normalize_tag_value(t) for t in raw_feed_tags) if n}
        # A pinned tag the publisher does NOT also ship gets no filter arrows:
        # they toggle a feed tag_filter rule keyed on the publisher's tags, and
        # filtering on a word this feed never publishes does nothing. One the
        # publisher does ship is an ordinary feed tag that happens to be pinned,
        # so it keeps them.
        pinned_only_tags = [t for t in entry_pinned_tags if t not in _publisher_tags]
        _manual_now = {normalize_tag_value(t) for t in (manual_tags or [])}
        feed_tag_suggestions = []
        for raw_tag in [*entry_pinned_tags, *raw_feed_tags]:
            normalized = normalize_tag_value(raw_tag)
            if not normalized or normalized in feed_tag_suggestions:
                continue
            # Once a pinned-only tag is applied there is nothing left to suggest
            # — the tag is already on the post and shown as a real tag chip. A
            # publisher tag still shows, because its chip is a filter control.
            if normalized in pinned_only_tags and normalized in _manual_now:
                continue
            feed_tag_suggestions.append(normalized)
        # Current +/- state of the feed's tag_filter rule, so active signs render lit.
        feed_tag_filter_signs: dict[str, str] = {}
        feed_tag_inherited_signs: dict[str, str] = {}
        _author_token = author_filter_token(getattr(entry, "authors_str", None))
        if feed_tag_suggestions or _author_token:
            with get_meta_connection() as _rule_conn:
                _rule = get_feed_tag_filter_rule(_rule_conn, str(entry.feed_url))
                feed_tag_inherited_signs = get_inherited_tag_filter_signs(_rule_conn, str(entry.feed_url))
            if _rule:
                _req, _good, _exc = parse_tag_filter_spec(str(_rule["keyword"] or ""))
                feed_tag_filter_signs = {t: "+" for t in (_req | _good)} | {t: "-" for t in _exc}

        # Check if entry is saved
        is_saved = False
        with get_meta_connection() as conn:
            row = conn.execute(
                "SELECT 1 FROM saved_entries WHERE feed_url = ? AND entry_id = ?",
                (feed_url, entry_id),
            ).fetchone()
            is_saved = bool(row)

        with get_meta_connection() as _prefs_conn:
            _disp = get_feed_display_prefs(_prefs_conn, str(entry.feed_url))
        _show_lead_in_article = bool(_disp.get("show_lead_image_in_article", 1))

        lead_image_url, _pending_lead_image = _resolve_article_lead_image(entry, video_id, _show_lead_in_article)
        # The lead image is what the article actually displays, and it comes from
        # the lead-image cache rather than the body — so a DeviantArt URL here
        # needs the same expiry check the body gets, or the post shows nothing
        # while its (re-signed) body image would have been fine.
        if isinstance(lead_image_url, str) and "wixmp" in lead_image_url:
            _fresh_lead = _resign_expired_deviantart_url(lead_image_url, entry_id)
            if _fresh_lead and _fresh_lead != lead_image_url:
                lead_image_url = _fresh_lead
                lead_image_service.persist_lead_image_async(feed_url, entry_id, _fresh_lead)

        # Remove inline images whose src URL is a logo, tracker, or avatar — these
        # are typically brand assets or analytics pixels embedded by feed publishers
        # and should not appear as article visuals in the reader.
        if isinstance(content_html, str):

            def _strip_bad_img(m: re.Match) -> str:
                src_m = re.search(r'\bsrc=(?:"([^"]*)"|\x27([^\x27]*)\x27)', m.group(0), re.IGNORECASE)
                if not src_m:
                    return m.group(0)
                src = src_m.group(1) or src_m.group(2) or ""
                _src_parsed = urlparse(src)
                # Check avatar hints against path only — CDN domains like
                # "googleusercontent.com" contain "user" and would false-positive
                # on the full URL.
                if (
                    lead_image_service._LOGO_URL_PATTERNS.search(src)
                    or lead_image_service._TRACKER_URL_PATTERNS.search(_src_parsed.netloc)
                    or lead_image_service._TRACKER_URL_PATTERNS.search(_src_parsed.path)
                    or lead_image_service._AVATAR_HINT_PATTERNS.search(_src_parsed.path)
                    or lead_image_service._SITE_CHROME_DOMAIN_PATTERNS.search(_src_parsed.netloc)
                    or lead_image_service._SITE_CHROME_PATH_PATTERNS.search(_src_parsed.path)
                ):
                    return ""
                return m.group(0)

            content_html = re.sub(r"<img\b[^>]*/?>", _strip_bad_img, content_html, flags=re.IGNORECASE) or None

        # Webcomic nav button cleanup — strip <a><img alt="Previous/Next/..."></a> combos
        # that comic feeds embed before and after the strip image for in-site navigation.
        if isinstance(content_html, str) and _COMIC_NAV_ALT_RE.search(content_html):
            content_html = _strip_comic_nav_images(content_html) or None

        # Strip orphaned <source> elements (meaningless outside <picture>/<video>,
        # but some feeds — notably Substack — include them for WebP alternatives).
        # Safe to remove: the <img> fallback still renders the image.
        # Skip <source> elements inside <video> or <picture> — those are required.
        if isinstance(content_html, str) and "<source" in content_html.lower():
            try:
                from bs4 import BeautifulSoup as _BS4

                _src_soup = _BS4(content_html, "html.parser")
                for _src_tag in _src_soup.find_all("source"):
                    if getattr(_src_tag.parent, "name", None) not in ("video", "picture"):
                        _src_tag.decompose()
                content_html = (_src_soup.body.decode_contents() if _src_soup.body else str(_src_soup)).strip() or None
            except Exception:
                content_html = re.sub(r"<source\b[^>]*/?>", "", content_html or "", flags=re.IGNORECASE) or None

        # Steam CDN serves localized images as /hash/english.png (404s for non-English)
        # with an onerror fallback to /hash.png (200). Strip the language subfolder so
        # the image loads directly without needing the stripped onerror handler.
        if isinstance(content_html, str) and "clan.fastly.steamstatic.com" in content_html:
            content_html = re.sub(
                r"(https://clan\.fastly\.steamstatic\.com/images/\d+/[a-f0-9]{40})/[a-z]+(\.\w+)",
                r"\1\2",
                content_html,
                flags=re.IGNORECASE,
            )

        # Flat-text feeds (e.g. forum/tracker news) use <br> for line breaks with no
        # <p> structure.  When there are at least 3 <br> tags and more <br>s than <p>s,
        # promote consecutive <br> runs to paragraph breaks so the entry renders as
        # readable paragraphs instead of a wall of text.
        if isinstance(content_html, str):
            # Some feeds (e.g. some trackers) double-encode <br> as &lt;br&gt; inside CDATA;
            # normalize those to actual <br> tags before the conversion check.
            if content_html.lower().count("&lt;br") >= 3:
                content_html = re.sub(r"&lt;br\s*/?\s*&gt;", "<br>", content_html, flags=re.IGNORECASE)
            _br_count = content_html.lower().count("<br")
            _p_count = content_html.lower().count("<p")
            if _br_count >= 3 and _br_count > _p_count:
                content_html = re.sub(
                    r"(?:<br\s*/?>\s*){2,}",
                    "</p><p>",
                    content_html,
                    flags=re.IGNORECASE,
                )
                if not content_html.startswith("<p"):
                    content_html = "<p>" + content_html
                if not content_html.rstrip().endswith("</p>"):
                    content_html = content_html + "</p>"

        # Strip Substack React UI chrome elements that render as broken/orphaned
        # widgets without Substack's CSS (expand buttons, pencraft layout divs).
        if isinstance(content_html, str) and ("image-link-expand" in content_html or "pencraft" in content_html):
            try:
                from bs4 import BeautifulSoup

                _cs = BeautifulSoup(content_html, "html.parser")
                for _junk in _cs.select(".image-link-expand, [class*=pencraft]"):
                    _junk.decompose()
                content_html = (_cs.body.decode_contents() if _cs.body else str(_cs)).strip() or None
            except Exception:
                pass

        # A feed that ships its own full HTML floats images with its site's classes (WordPress
        # alignleft/alignright), and a reader never ships that stylesheet — so the class survived and did
        # nothing, and every image landed full-width with the text meant to wrap beside it pushed below.
        # lift_float_classes was only ever applied to source-page extraction, so scraped articles floated
        # correctly and feed-supplied ones did not. Lifted here, before the opener strip, so the float guard
        # in _strip_lead_image_opener can see a class-based float as the inline style it checks for.
        if isinstance(content_html, str) and content_html:
            content_html = html_sanitize.lift_float_classes(content_html)

        image_title_text, _in_feed_title_is_lead_img, _persisted_alt, _persisted_title = _initial_image_caption(
            content_html, entry, lead_image_url
        )

        # Strip the opener thumbnail and dedup against the remaining content.
        # Only when we will actually display the lead image at the top — if
        # show_lead_image_in_article is off the image should stay in place.
        # Order matters: strip the leading <img> first, then check if the lead
        # image URL still appears in what remains.  This prevents the case where
        # the lead image IS the opener thumbnail (e.g. comicsthumbs) from being
        # incorrectly suppressed just because it appears at the top of content.
        # Keep what the lead image RESOLVED to, before the dedup below decides
        # whether to render it separately. The two are different questions, and
        # conflating them blanks thumbnails: everything the dedup does — a lead
        # already shown inline, a floated opener left in the flow — sets
        # lead_image_url to None meaning "don't draw it twice", not "this entry
        # has no image". That None used to be what got persisted to
        # entry_lead_images, which is where the LIST reads its thumbnail from.
        _resolved_lead_for_cache = lead_image_url
        # Captured before the strip: distinguishes "body never had a picture"
        # (mahonoir) from "body had one and it was just deduped against the
        # lead" (misfile.com) for the webcomic injector below — both look
        # identical AFTER the strip (no <img> left either way).
        _body_had_image_before_strip = bool(content_html and _HAS_IMG_RE.search(content_html))
        # Same idea, but a count rather than a bool: the inject_source_images gate
        # further below judges "is this body already image-rich" against whatever
        # <img> count remains AFTER this strip removes a matching lead-image opener
        # -- so a normal image-rich post (lead + one more body image) reads as only
        # one remaining image post-strip and wrongly re-triggers the expensive
        # source-page fetch this gate exists to avoid. The gate takes max(this,
        # its own later count) so a strip never makes an already-rich post look thin.
        _body_img_count_before_strip = len(re.findall(r"<img\b", content_html or "", re.IGNORECASE))
        # Inline panels tile before the opener strip, which would otherwise pull panel 1 out as the hero above the grid.
        _tile_after_strip = False
        if isinstance(content_html, str) and lead_image_service._is_feed_webcomic(str(entry.feed_url)):
            if _show_lead_in_article:
                content_html, _tiled = _tile_inline_webcomic_panels(_strip_ad_images(content_html) or "")
                # Only a lead that is one of the tiled panels is drawn by the grid; any other is a separate hero and stays.
                if _tiled and lead_image_url and lead_image_url in {_img_src(i) for i in _IMG_TAG_RE.findall(content_html)}:
                    lead_image_url = None
            else:
                # A hidden lead still has to come out of the body, and that strip only finds an opener, so it goes first.
                _tile_after_strip = True
        content_html, lead_image_url = _strip_lead_image_opener(content_html, lead_image_url, str(entry.feed_url), _show_lead_in_article)
        if _tile_after_strip and isinstance(content_html, str):
            content_html = _tile_inline_webcomic_panels(_strip_ad_images(content_html) or "")[0]
        # AFTER the strip, deliberately: injecting first would hand the new <img>
        # to _strip_lead_image_opener as the body's opener and it would be
        # removed again.
        content_html, lead_image_url = _inject_webcomic_panel_into_bodyless_entry(
            content_html,
            entry,
            str(entry.feed_url),
            lead_image_url,
            body_had_image=_body_had_image_before_strip,
            show_lead_in_article=_show_lead_in_article,
        )
        content_html, lead_image_url = _inject_tapas_episode_panels(content_html, entry, str(entry.feed_url), lead_image_url)
        content_html, lead_image_url = _inject_webtoons_episode_panels(content_html, entry, str(entry.feed_url), lead_image_url)
        content_html = _collapse_block_spacers(_strip_ad_images(content_html))

        # Fallback: check the alt text on the main image on the source page.
        # Covers feeds that only supply a thumbnail in the content (e.g. Wilde Life)
        # where the alt text lives on the full-size img on the article page.
        # Also runs when the in-feed title came from a different image than the lead
        # (e.g. Oglaf story thumbnail in feed vs. full comic from source page) and no
        # persisted result exists yet — source scrape may find a better hovertext.
        # If the source HTML is already in-memory (same session as the lead-image
        # fetch) run synchronously — no network cost.  Otherwise queue a background
        # fetch so the render doesn't block on a slow HTTP GET; alt text will appear
        # on the next open once the background thread stores it in the DB.
        _needs_source_scrape = image_title_text is None or (not _in_feed_title_is_lead_img and not _persisted_alt and not _persisted_title)
        _is_wc_feed = lead_image_service._is_feed_webcomic(str(entry.feed_url))
        if _needs_source_scrape and lead_image_url and entry.link:
            if entry.link in lead_image_service._source_html_cache:
                _fa, _ft = lead_image_service.fetch_entry_image_caption(entry.link, lead_image_url=lead_image_url, is_webcomic=_is_wc_feed)
                _fa = (re.sub(r"<[^>]+>", "", _fa).strip() or None) if _fa else None
                _ft = (re.sub(r"<[^>]+>", "", _ft).strip() or None) if _ft else None
                if _fa or _ft:
                    image_title_text = _ft or _fa  # title preferred for auto display
                    lead_image_service.persist_image_alt_async(str(entry.feed_url), str(entry.id), _fa, title_text=_ft)
            else:
                # Source HTML isn't cached — fetching it is a slow network GET (several
                # seconds for some og_scrape hosts, e.g. mynorthwest). Do it fully in
                # the background: queue_source_html_fetch (given feed_url/entry_id/
                # lead_image_url) fetches the page AND extracts + persists the caption,
                # so it appears on the next open from the DB. Previously the render
                # thread blocked up to 3s here waiting for that fetch just to maybe show
                # the caption on this first open — a big first-open stall for no durable
                # benefit. Cache-first / defer, matching the async lead-image fetch.
                lead_image_service.queue_source_html_fetch(
                    entry.link,
                    feed_url=str(entry.feed_url),
                    entry_id=str(entry.id),
                    lead_image_url=lead_image_url,
                )

        # Drop trivially generic alt texts that add no information (e.g. Bootstrap
        # class names used as alt values, single-word placeholders, or navigation
        # labels from wrongly-selected nav thumbnails like "Previous" / "Next").
        image_title_text = _suppress_junk_caption(image_title_text, entry)

        # The per-feed caption preference has to be applied BEFORE the inline
        # injection below, not after it. Applied after, it could not suppress
        # anything: the injection writes the caption into content_html and then
        # clears image_title_text, so the later call had nothing left to act on
        # and caption_source="none" left the caption on screen. It only ever
        # worked for the template caption, i.e. entries WITH a separate hero.
        image_title_text = _apply_caption_source_pref(image_title_text, _disp, entry, content_html)

        # Inject image_title_text as alt attribute on the first <img> in content_html
        # and insert a caption <p> immediately after it so it appears inline under
        # the image rather than at the bottom of the article.
        if image_title_text and not lead_image_url and isinstance(content_html, str):
            _caption_injected = False

            _title_str: str = image_title_text  # narrowed by outer `if image_title_text`

            def _inject_alt(m: re.Match) -> str:
                nonlocal _caption_injected
                tag = m.group(0)
                if re.search(r"\balt\s*=", tag, re.IGNORECASE):
                    tag = re.sub(
                        r'(\balt\s*=\s*)(?:"[^"]*"|\x27[^\x27]*\x27)',
                        lambda a: a.group(1) + '"' + _title_str.replace('"', "&quot;") + '"',
                        tag,
                        count=1,
                        flags=re.IGNORECASE,
                    )
                else:
                    tag = tag[:-1] + ' alt="' + _title_str.replace('"', "&quot;") + '"' + tag[-1]
                _caption_injected = True
                caption = f'<p class="entry-image-title-text">{html.escape(_title_str)}</p>'
                return tag + caption

            content_html = re.sub(r"<img\b[^>]*/?>", _inject_alt, content_html, count=1, flags=re.IGNORECASE)
            if _caption_injected:
                # Caption is now inline in content_html; clear it so the template
                # doesn't also render it at the bottom of the article.
                image_title_text = None

        # SMBC: append the bonus panel image from the source page.
        if entry.link and "smbc-comics.com" in (entry.link or ""):
            _bonus_url = lead_image_service.fetch_smbc_bonus_panel_url(entry.link)
            if _bonus_url:
                _bonus_img = f'<p><img src="{html.escape(_bonus_url, quote=False)}" alt="Bonus panel" /></p>'
                content_html = (content_html or "") + _bonus_img
            # The SMBC feed wraps the comic <img> in an <a> with the text
            # "Click here to go see the bonus panel!".  After the lead-image
            # opener strip removes the <img>, that link text is left as an
            # orphaned anchor.  It's redundant because we show the panel inline.
            if isinstance(content_html, str):
                content_html = (
                    re.sub(
                        r"<a\b[^>]*>(?:\s*<br\s*/?>\s*)*Click here to go see the bonus panel!\s*</a\s*>",
                        "",
                        content_html,
                        flags=re.IGNORECASE,
                    ).strip()
                    or None
                )

        # Persist off the request thread (and skip when unchanged) so an open
        # never blocks on the meta-DB writer held by the background backfill.
        # Skip for webcomic feeds: their cached lead is the single-pane source
        # preview used as the list thumbnail, while the article shows the full
        # inline strip — persisting the article's full image here would clobber
        # the preview thumbnail. The webcomic source-scrape owns that cache.
        #
        # Persist what the lead image RESOLVED to, not what survived the dedup —
        # see _resolved_lead_for_cache above. The list thumbnail reads this cache,
        # so persisting the dedup's None takes the thumbnail away from every post
        # whose image is shown inline instead of hoisted.
        _persist_strategy, _, _ = lead_image_service.get_feed_strategy(str(entry.feed_url))
        if _persist_strategy != "webcomic":
            lead_image_service.persist_lead_image_async(str(entry.feed_url), str(entry.id), _resolved_lead_for_cache)

        # If this entry is starred and the archive worker has captured assets,
        # swap inline image URLs to the local /starred-asset route so the
        # entry remains readable even if the source goes down.
        if is_saved:
            asset_map = starred_archive_service.get_entry_asset_map(str(entry.feed_url), str(entry.id))
            if asset_map:
                if isinstance(content_html, str) and content_html:
                    content_html = starred_archive_service.rewrite_html_assets(content_html, asset_map, STARRED_ASSET_URL_PREFIX)
                if lead_image_url and lead_image_url in asset_map:
                    lead_image_url = f"{STARRED_ASSET_URL_PREFIX}{asset_map[lead_image_url]}"

        # Source-page image gallery (opt-in per feed) — runs before the hotlink/
        # no-referrer pass below so injected images are proxied/referrer-stripped too.
        # Gated on the body actually being thin on images: both paths exist for
        # a feed whose posts carry 0 (paizo-style prose) or 1 (a webcomic's
        # gallery-append case) image of their own, per _source_article_body's
        # own docstring — not for a normal, image-rich post. Without this check
        # every entry on an inject_source_images feed paid a synchronous
        # source-page fetch regardless, which is merely wasted work on a fast
        # host but, on one that needs FlareSolverr to pass its bot challenge
        # (play.nobleknight.com), turned into a 6+ second render for a post
        # that already had four images in place — reported live as "missing
        # lead img & thumb, takes a long time to load" (the images WERE there;
        # the render was just slow enough to look stuck).
        if (
            _disp.get("inject_source_images")
            and entry.link
            and max(_body_img_count_before_strip, len(re.findall(r"<img\b", content_html or "", re.IGNORECASE))) < 2
        ):
            # Prefer the source article itself: it carries the images IN PLACE.
            # The gallery append remains the fallback for pages readability
            # cannot make sense of, so a feed already relying on it (a webcomic
            # whose page is one image and no prose) does not regress.
            _sourced = _source_article_body(entry)
            if _sourced:
                content_html = _sourced
            else:
                content_html = _inject_source_gallery(content_html, entry, lead_image_url)

        # Suppress the Referer on inline body images so hotlink-protected hosts
        # serve the real image instead of a placeholder, and route known
        # hotlink hosts (e.g. nanolx.org) through the /api/img proxy so a
        # browser-cached placeholder under the original URL is bypassed. Skip
        # locally-served starred assets (same-origin, no hotlink concern).
        # DeviantArt signs mature deviations' images for ~15 minutes, so a stored
        # URL is usually dead by the time it is read. Re-sign on open, before the
        # proxy rewrite below — the proxy's byte cache then holds the image for
        # good, so this costs one API call per image, not per view.
        if isinstance(content_html, str) and content_html:
            content_html = _resign_expired_deviantart_images(content_html, feed_url, entry_id)

        if isinstance(content_html, str) and content_html and not is_saved:
            if proxy_body_images_enabled():
                # Every remote src already routes through /api/img, so the
                # named-host hotlink rewrite below is redundant. The onerror
                # fallback is NOT: a src that is dead at the source (an
                # already-expired signed URL, say) still needs it, since the
                # fallback's job on an already-proxied image is to hide a
                # genuine failure (display:none), not just to retry.
                content_html = proxy_all_body_images(content_html)
            else:
                content_html = proxy_hotlink_images(content_html)
            content_html = add_no_referrer_to_images(content_html)
            content_html = add_img_proxy_fallback(content_html)

        if not _show_lead_in_article:
            lead_image_url = None
            # The caption describes the lead image. With the image deliberately
            # not shown, the template's no-lead branch rendered it alone at the
            # very BOTTOM of the article — alt text for a picture that is not on
            # screen, hundreds of words from anything it could refer to.
            image_title_text = None

        # (caption_source is applied above, before the inline injection. Applying
        # it a second time here would re-read alt/title from storage and render
        # the caption twice — once inline, once from the template.)

        # Fold the channel link through declared host migrations first, so a feed
        # whose channel <link> still names a dead domain can't rebase a correct
        # entry link back onto it (see _rebase_proxy_entry_link, the list view).
        _channel_link = _rewrite_url_host(
            getattr(entry.feed, "link", None) if hasattr(entry, "feed") else None,
            get_dedupe_host_aliases(),
        )
        # Drop a feed-supplied javascript:/data:/etc link before it reaches the
        # entry pane's href / data-source-url attributes (see safe_link_url).
        _display_link = html_sanitize.safe_link_url(
            _rebase_proxy_entry_link(entry.link or _derived_entry_link(entry), feed_url, _channel_link)
        )

        # Suppress summaries that consist entirely of img tags with no text (e.g. xkcd,
        # Deathbulge).  After the lead image is shown above the content, rendering the
        # raw <img> tag in a <pre> block would show it as literal HTML text.
        _summary = entry.summary
        if isinstance(_summary, str):
            # Dinosaur Comics: strip site-nav table and prev/next row from the summary
            if "qwantz.com" in (entry.link or ""):
                _summary = _strip_qwantz_nav(_summary)
            # Webcomic nav button images in summary (same as content_html path above)
            if _COMIC_NAV_ALT_RE.search(_summary):
                _summary = _strip_comic_nav_images(_summary)
            _summary_no_imgs = re.sub(r"<img\b[^>]*/?>", "", _summary, flags=re.IGNORECASE).strip()
            _summary_text_only = re.sub(r"<[^>]+>", " ", _summary_no_imgs)
            _summary_text_only = html.unescape(re.sub(r"\s+", " ", _summary_text_only)).strip()
            if not _summary_text_only:
                _summary = None

        # Same table/column list_entries_for_feeds batches into _locked_until_map
        # for a whole view; a single-row lookup here is fine since the pane only
        # ever resolves one entry. Surfaced regardless of hide_locked_comics —
        # that preference only controls whether the entry appears in the list at
        # all, not whether a reader who left it off sees a real image here.
        _locked_until = lead_image_service.get_locked_until(feed_url, entry_id)
        _is_locked = _locked_until is not None and _locked_until > time.time()

        _total_ms = int((time.monotonic() - _t0) * 1000)
        if _total_ms > 500:
            LOGGER.info("[perf] entry_detail: total=%dms %s", _total_ms, entry_id)
        return {
            "feed_url": entry.feed_url,
            "id": entry.id,
            "title": _display_title(entry) or entry.title,
            # Two forms of the same string: title_html renders the feed's inline
            # formatting (<em>), title_plain drops those tags for the title=
            # attribute and any other context that cannot render HTML. Everything
            # not on the tiny allowlist stays literal, so `std::vector<T>` in a
            # C++ post title survives — see sanitize_inline_title.
            "title_html": html_sanitize.sanitize_inline_title(_display_title(entry) or entry.title or ""),
            "title_plain": html_sanitize.title_plain_text(_display_title(entry) or entry.title or ""),
            "link": _display_link,
            "summary": _summary,
            "content_html": content_html,
            "lead_image_url": _lead_image_display_url(lead_image_url, _disp.get("image_size_rule")),
            "show_lead_in_article": _show_lead_in_article,
            "is_locked": _is_locked,
            "locked_until_ts": _locked_until if _is_locked else None,
            "locked_until_display": (
                format_datetime_for_ui(datetime.fromtimestamp(_locked_until, tz=timezone.utc)) if _is_locked else None
            ),
            "katex_dollar_math": bool(_disp.get("katex_dollar_math", 0)),
            "show_as_thumb": bool(_disp.get("show_lead_image_as_thumb", 1)) and not _disp.get("feed_thumbnail_url"),
            # Webcomic feeds show the FULL strip in the article but keep the
            # single-pane scraped preview as the list thumbnail — so don't let the
            # client sync the list row's thumbnail to the article lead image on open.
            "sync_list_thumb": _persist_strategy != "webcomic",
            "image_title_text": image_title_text,
            "duration_seconds": duration_seconds,
            "duration_display": duration_display,
            # The post's own video (YouTube feeds only); the pane's embed "Add to playlist" button is limited to it.
            "video_id": video_id,
            "feed_title": entry.feed_resolved_title or entry.feed_url,
            "post_timestamp": published_dt.isoformat() if published_dt else None,
            "received_timestamp": entry.added.isoformat() if entry.added else None,
            "post_display": format_datetime_for_ui(published_dt),
            "received_display": format_datetime_for_ui(entry.added),
            "author": author_name,
            "read": bool(entry.read),
            "saved": is_saved,
            "content_edited": content_edited,
            # See the post-list builder: Re-fetch follows the entry, not the feed.
            "captured": str(getattr(entry, "added_by", "") or "") == "user",
            "manual_tags": manual_tags,
            # Starred OR tagged, same rule (and same name) the list rows carry, so
            # the re-fetch hatch keys on one field in both templates rather than
            # each re-deriving it.
            "kept": bool(is_saved or manual_tags),
            "manual_tags_text": " ".join(manual_tags),
            "feed_tag_suggestions": feed_tag_suggestions,
            "pinned_feed_tags": entry_pinned_tags,
            "pinned_only_feed_tags": pinned_only_tags,
            "feed_tag_chips_collapsed": FEED_TAG_CHIPS_COLLAPSED,
            "feed_tag_filter_signs": feed_tag_filter_signs,
            "feed_tag_inherited_signs": feed_tag_inherited_signs,
            "author_filter_token": _author_token,
            "da_watch_artist": da_watch_artist,
            "feed_icon_url": get_favicon_url(entry.feed_url, getattr(entry.feed, "link", None) if hasattr(entry, "feed") else None),
            "pending_lead_image": _pending_lead_image,
            "audio_feed_suggestion": _audio_feed_suggestion,
        }


def mark_feeds_as_read(feed_urls: set[str]) -> tuple[int, str | None]:
    """Mark every unread entry in *feed_urls* read. Returns (count, batch
    timestamp). The whole batch shares one read_at stamp, which doubles as the
    undo token for /entries/undo-mark-read."""
    if not feed_urls:
        return 0, None

    to_sync: list[tuple[str, str]] = []
    with get_reader() as reader:
        for feed_url in feed_urls:
            for entry in reader.get_entries(feed=feed_url, read=False):
                # A premiere that hasn't aired yet shouldn't be swallowed by a
                # blanket mark-all-as-read — it hasn't delivered its content.
                if _is_youtube_unpremiered(entry):
                    continue
                reader.mark_entry_as_read((entry.feed_url, entry.id))
                to_sync.append((entry.feed_url, entry.id))

    when = None
    if to_sync:
        when = datetime.now().isoformat()
        with get_meta_connection() as conn:
            conn.executemany(
                """
                INSERT INTO entry_read_state (feed_url, entry_id, read_at)
                VALUES (?, ?, ?)
                ON CONFLICT(feed_url, entry_id) DO UPDATE SET read_at = excluded.read_at
                """,
                [(fu, eid, when) for fu, eid in to_sync],
            )
    return len(to_sync), when


def _mark_entries_as_read_for_view(
    feed_urls: set[str],
    *,
    sort_by: str | None = None,
    sort_dir: str | None = None,
    read_filter: str | None = None,
    star_only: str | int | bool | None = None,
    tag: str | None = None,
    older_than_cutoff: datetime | None = None,
    post_filter: Callable[[dict], bool] | None = None,
    loaded_before: float | None = None,
    search_query: str | None = None,
) -> tuple[int, str | None]:
    """Mark as read only the entries the current filtered view actually shows —
    tag, star_only, read_filter, and every hide_* display preference
    list_entries_for_feeds itself applies (hide_locked_comics, hide_shorts,
    hide_paywalled, hide_members_only, hide_unpremiered) — not every raw
    unread entry in feed_urls regardless of what the view is hiding.

    Shared by /folders/mark-read, /feeds/mark-read, and
    /entries/mark-older-than-read, each of which used to ignore the tag/
    star_only/read_filter params their own forms already submitted (accepted
    only to round-trip into the redirect URL) and mark every unread entry in
    scope instead — so marking a filtered/starred/tagged view as read could
    silently mark entries the filter was hiding. Only /entries/mark-range-read
    ("Read above/below") was already scoped correctly, via this same
    list_entries_for_feeds call.

    older_than_cutoff, when given, additionally requires the same effective
    date (published/updated/added, via entry_effective_date) the list itself
    displays and greys on to be before it — the "Older than N days" action's
    own scoping, layered on top of the view filter rather than replacing it.

    post_filter is the posts list's "Filter this view" predicate; it needs enriched rows (chip tags, titles), so the
    cheap enrich=False pass is used only without it.

    loaded_before (epoch seconds, the moment the list page was rendered) leaves out entries stored after it: a post that
    arrived while the list sat open was never in front of the user, so a blanket mark-read must not swallow it.

    search_query is the view's server-side search (the topbar box / DA artist link): only its matches are marked.
    """
    if not feed_urls:
        return 0, None
    # list_entries_for_feeds(enrich=False) is used only to compute WHICH
    # entries the current tag/star_only/read_filter/hide_* view actually shows
    # — cheaply (no thumbnails/tags/premiere-prefix work) — not as the source
    # of the entries themselves, since that shape drops post_timestamp and
    # everything else an enriched caller would get. The actual read/date
    # checks below run against raw reader entries, same as before this scoping
    # existed.
    allowed_ids = {
        (post["feed_url"], post["id"])
        for post in list_entries_for_feeds(
            feed_urls,
            limit=_RANGE_READ_LIMIT,
            sort_by=sort_by or "post",
            sort_dir=sort_dir or "asc",
            read_filter=read_filter or "all",
            star_only=normalize_star_only(star_only),
            selected_tag=tag,
            search_query=search_query,
            enrich=post_filter is not None,
        )
        if post_filter is None or post_filter(post)
    }
    if not allowed_ids:
        return 0, None

    to_sync: list[tuple[str, str]] = []
    added_cutoff = datetime.fromtimestamp(loaded_before, tz=timezone.utc) if loaded_before and math.isfinite(loaded_before) else None
    with get_reader() as reader:
        for feed_url in feed_urls:
            for entry in reader.get_entries(feed=feed_url, read=False):
                if (entry.feed_url, entry.id) not in allowed_ids:
                    continue
                if (
                    added_cutoff is not None
                    and entry.added is not None
                    and entry.added.replace(tzinfo=entry.added.tzinfo or timezone.utc) > added_cutoff
                ):
                    continue
                if older_than_cutoff is not None:
                    date = entry_effective_date(entry)
                    if date is None:
                        continue
                    if date.tzinfo is None:
                        date = date.replace(tzinfo=timezone.utc)
                    if date >= older_than_cutoff:
                        continue
                # A premiere that hasn't aired yet shouldn't be swallowed by a
                # blanket mark-read sweep, even when hide_unpremiered itself is
                # off (matching every other mark-read entry point's
                # unconditional exemption) — it hasn't delivered its content.
                if _is_youtube_unpremiered(entry):
                    continue
                try:
                    reader.mark_entry_as_read((entry.feed_url, entry.id))
                except Exception:
                    continue
                to_sync.append((entry.feed_url, entry.id))

    when = None
    if to_sync:
        when = datetime.now().isoformat()
        with get_meta_connection() as conn:
            conn.executemany(
                """
                INSERT INTO entry_read_state (feed_url, entry_id, read_at)
                VALUES (?, ?, ?)
                ON CONFLICT(feed_url, entry_id) DO UPDATE SET read_at = excluded.read_at
                """,
                [(fu, eid, when) for fu, eid in to_sync],
            )
    return len(to_sync), when


def _is_youtube_host(host: str) -> bool:
    """True for youtube.com / youtu.be and their subdomains (exact suffix match,
    not substring — "youtube.com.evil.com" must not pass)."""
    host = (host or "").lower().rstrip(".")
    return host in ("youtube.com", "youtu.be") or host.endswith(".youtube.com") or host.endswith(".youtu.be")


def normalize_youtube_feed_url(feed_url: str) -> str:
    """Normalize various YouTube URLs (channel, handle, short links, video pages)
    to the canonical channel feed URL when possible.

    Returns the original URL if normalization fails or is not applicable.
    """
    try:
        parsed = urlparse(feed_url)
        host = (parsed.netloc or "").lower()

        # Already a YouTube feed
        if _is_youtube_host(host) and parsed.path and "feeds/videos.xml" in parsed.path:
            return feed_url

        # Direct /channel/UC... path
        m = re.search(r"/channel/(UC[0-9A-Za-z_-]+)", parsed.path or "")
        if m:
            channel_id = m.group(1)
            normalized = f"https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
            logging.info("Normalized YouTube channel URL %s -> %s", feed_url, normalized)
            return normalized

        # If it's any youtube domain (including youtu.be), try fetching page and extracting channelId
        if _is_youtube_host(host):
            try:
                # SSRF-guarded: the host check above stops lookalike domains, and
                # safe_get revalidates every redirect hop (youtu.be bounces to
                # youtube.com; a crafted link must not bounce somewhere internal).
                with url_guard.build_client(timeout=6.0, headers={"User-Agent": "Lectio/1.0"}) as _yc:
                    resp = url_guard.safe_get(_yc, feed_url)
                text = resp.text
            except Exception:
                return feed_url

            # meta itemprop channelId
            m = re.search(r'<meta\s+itemprop=["\']channelId["\']\s+content=["\'](UC[0-9A-Za-z_-]+)["\']', text)
            if not m:
                # look for JSON field "channelId":"UC..."
                m = re.search(r'"channelId"\s*[:=]\s*"(UC[0-9A-Za-z_-]+)"', text)

            if m:
                channel_id = m.group(1)
                normalized = f"https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
                logging.info("Resolved YouTube %s -> channel id %s", feed_url, channel_id)
                return normalized
            # Fallback: look for any /channel/UC... path in the HTML (appears in some pages)
            m = re.search(r"/channel/(UC[0-9A-Za-z_-]+)", text)
            if m:
                channel_id = m.group(1)
                normalized = f"https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
                logging.info("Resolved YouTube %s -> channel id %s (from /channel/ in HTML)", feed_url, channel_id)
                return normalized
    except Exception:
        return feed_url

    return feed_url


_FORMAT_SELECTOR_PARAMS = frozenset({"alt", "type", "feed"})
# Both the param name AND value must match a known-safe combo before
# normalize_feed_url strips it -- an allowlist, not a blanket "ignore this
# query param" rule, so a hypothetical ?type=news vs ?type=sports category
# selector is untouched (its values aren't in this set) while tosecdev.org's
# ?type=atom/?type=rss and paizo.com's ?feed=json1/?feed=rss (reported
# 2026-08-10) fold into one canonical form and pick up the existing
# same_folder/cross_folder auto-handling, same as Blogger's ?alt=rss always
# has. Genuinely unrecognized query differences still fall to the
# never-pre-checked query_pairs tier in get_feed_duplicates.
_FORMAT_SELECTOR_VALUES = frozenset({"rss", "rss2", "atom"})


def _is_format_selector_value(value: str) -> bool:
    # JSON Feed versioning is a moving target (json1 today; a hypothetical
    # json2/jsonfeed tomorrow), so this family is prefix-matched rather than
    # enumerated -- rss/atom aren't, since there's no equivalent versioned
    # naming pattern to future-proof there.
    return value in _FORMAT_SELECTOR_VALUES or value.startswith("json")


def _dedup_dismiss_key(urls: list[str]) -> str:
    """Canonical key for a duplicate-scan dismissal: the exact set of feed
    URLs shown in the group, order-independent. \\x1f can't appear in a URL,
    so it's a safe join separator."""
    return "\x1f".join(sorted(set(urls)))


_DOMAIN_ALIASES: dict[str, str] = {
    # old.reddit.com and www.reddit.com serve identical RSS content
    "old.reddit.com": "www.reddit.com",
    # Tapastic rebranded to Tapas; the old domain's feeds 404. Same path structure
    # (/rss/series/<id>), so rewriting the host keeps existing subscriptions working.
    "tapastic.com": "tapas.io",
    "www.tapastic.com": "tapas.io",
}


def assume_https_if_schemeless(url: str) -> str:
    """A pasted `www.example.com` or `example.com/feed` (no `://` at all) is
    assumed https — the common case, and the alternative is rejecting a paste
    a user would expect to just work. Shared by Add Feed's discovery route and
    Change URL, which both accept the same kind of paste; the add-feed
    dialog's own JS does the equivalent client-side (`normalizeInput`), with a
    hostname-shape check the server side doesn't need since a garbage result
    just fails to resolve to a feed downstream."""
    return "https://" + url if url and "://" not in url else url


def normalize_feed_url(feed_url: str) -> str:
    """Normalize a feed URL for consistent storage and deduplication.

    - Strips trailing slashes from paths longer than "/".
    - Strips format-selector query params (e.g. Blogger's ?alt=rss) that
      select a serialization format without changing the feed content, so
      the Atom and RSS variants of the same Blogger feed are treated as one.
    - Rewrites ArtStation subdomain feeds (username.artstation.com/rss) to the
      main-domain form (www.artstation.com/username.rss) which works for all
      usernames including those with underscores that fail TLS hostname validation.
    - Rewrites known domain aliases (e.g. old.reddit.com → www.reddit.com).
    - Other normalization (YouTube links) is handled separately.
    """
    import re as _re

    _as_m = _re.match(r"(https?)://([^.]+)\.artstation\.com/rss$", feed_url.strip(), _re.IGNORECASE)
    if _as_m:
        feed_url = f"{_as_m.group(1)}://www.artstation.com/{_as_m.group(2)}.rss"
    try:
        parsed = urlparse(feed_url)
        if parsed.netloc.lower() in _DOMAIN_ALIASES:
            feed_url = parsed._replace(netloc=_DOMAIN_ALIASES[parsed.netloc.lower()]).geturl()
            parsed = urlparse(feed_url)
        path = parsed.path
        if path and path != "/" and path.endswith("/"):
            path = path.rstrip("/")
        # Strip pure format-selector params (alt=rss, alt=atom, etc.) only when
        # at least one such param is actually present — avoids re-encoding the
        # query string (which would turn e.g. ":" into "%3A" unnecessarily).
        path_changed = path != parsed.path
        if parsed.query:
            from urllib.parse import parse_qsl, urlencode

            all_pairs = parse_qsl(parsed.query, keep_blank_values=True)
            kept = [(k, v) for k, v in all_pairs if not (k in _FORMAT_SELECTOR_PARAMS and _is_format_selector_value(v.lower()))]
            new_query = urlencode(kept)
            # Never strip a selector down to a bare homepage. On WordPress
            # `?feed=atom` at "/" IS the feed, not a serialization choice
            # applied to one — dropping it yields the site root, which is not a
            # feed at all. That matters beyond dedupe: importers run every
            # incoming URL through canonical_feed_url and then SUBSCRIBE to the
            # result (_canonicalize_item_feed_urls), so a homepage would be
            # stored as the feed and the entry tags/stars keyed to it.
            #
            # The upgrade/alternates path already refuses the same candidate for
            # the same reason (see test_root_level_feed_param_still_offers_alternates);
            # this is that guard applied where the canonical form is produced.
            #
            # Found 2026-08-11: 12 subscribed feeds canonicalized to a bare
            # homepage this way.
            collapses_to_homepage = path in ("", "/") and not new_query
            if len(kept) != len(all_pairs) and not collapses_to_homepage:
                feed_url = parsed._replace(path=path, query=new_query).geturl()
            elif path_changed:
                feed_url = parsed._replace(path=path).geturl()
        elif path_changed:
            feed_url = parsed._replace(path=path).geturl()
    except Exception:
        pass

    # Case-normalize scheme + host only. Per RFC 3986 the scheme and host are
    # case-insensitive, but the path and query ARE case-sensitive (e.g. YouTube
    # channel IDs, signed/base64 tokens, Reddit /user/Name), so they're left
    # untouched. Userinfo (rare in feeds) is also case-sensitive — preserved.
    try:
        parsed = urlparse(feed_url)
        netloc = parsed.netloc
        if "@" in netloc:
            userinfo, hostport = netloc.rsplit("@", 1)
            new_netloc = userinfo + "@" + hostport.lower()
        else:
            new_netloc = netloc.lower()
        if parsed.scheme != parsed.scheme.lower() or netloc != new_netloc:
            feed_url = parsed._replace(scheme=parsed.scheme.lower(), netloc=new_netloc).geturl()
    except Exception:
        pass
    return feed_url


def canonical_feed_url(raw_url: str) -> str:
    """Canonical form of a feed URL for storage/dedupe at import time.

    Composes the same normalization the interactive Add-Feed path uses
    (YouTube channel canonicalization + ``normalize_feed_url``: host aliases
    like old.reddit→www.reddit, format-selector params, trailing slashes,
    scheme/host case). Importers must run every incoming feed URL through this
    so a variant (old.reddit, a trailing ``?``, ``alt=rss``) attaches to an
    existing subscription instead of creating a duplicate: ``reader.add_feed``
    with ``exist_ok=True`` then merges into the existing feed, and any
    per-entry tags/stars key off the same canonical URL. Returns the stripped
    input unchanged if normalization fails.
    """
    url = (raw_url or "").strip()
    if not url:
        return url
    try:
        url = normalize_youtube_feed_url(url)
    except Exception:
        pass
    try:
        url = normalize_feed_url(url)
    except Exception:
        pass
    return url


def _is_subscribable_feed_url(url: str) -> bool:
    """True only for http(s) feed URLs supplied by users.

    The ``reader`` library natively fetches ``file://`` (used internally for
    scraped feeds), so an unrestricted user-supplied URL — via Add Feed, a
    discovered ``<link>``, or an OPML ``xmlUrl`` — could read arbitrary local
    files (``file:///etc/passwd``, another tenant's DB, ``.env``) on refresh.
    Restrict user-facing subscription to http/https; internal scraped feeds add
    their ``file://`` URLs through ``reader.add_feed`` directly, not here.
    """
    try:
        return urlparse(url).scheme in ("http", "https")
    except ValueError:
        return False


def add_feed_to_folder(feed_url: str, folder_id: int) -> str:
    feed_url = feed_url.strip()
    if not feed_url:
        raise ValueError("Feed URL is required.")
    if not _is_subscribable_feed_url(feed_url):
        raise ValueError("Only http:// and https:// feed URLs can be added.")

    # Normalize YouTube-like links into canonical channel feed URLs when possible.
    try:
        feed_url = normalize_youtube_feed_url(feed_url)
    except Exception:
        # If normalization fails for any reason, proceed with the original URL.
        pass

    # Strip trailing slashes for consistent deduplication.
    feed_url = normalize_feed_url(feed_url)

    # If the slash-normalized URL isn't yet in this folder but the un-normalized
    # variant already is, reuse the existing URL to prevent duplicates.
    with get_meta_connection() as conn:
        existing = conn.execute(
            "SELECT feed_url FROM folder_feeds WHERE folder_id = ? AND (feed_url = ? OR feed_url = ?)",
            (folder_id, feed_url, feed_url + "/"),
        ).fetchone()
    if existing and existing[0] != feed_url:
        feed_url = existing[0]

    with get_reader() as reader:
        reader.add_feed(feed_url, exist_ok=True)

    # Re-adding a feed that already exists in reader as disabled (exist_ok=True
    # leaves its state untouched) must re-enable it — otherwise the feed stays
    # excluded from the sidebar tree while its old entries still surface in the
    # posts list, which reads as "added but missing".
    enable_feed(feed_url)

    # Re-subscribing a kept-but-unsubscribed feed clears its kept state so it
    # rejoins the tree/All Feeds instead of staying hidden in the Kept view.
    with get_meta_connection() as conn:
        conn.execute("DELETE FROM kept_feeds WHERE feed_url = ?", (feed_url,))

    with get_meta_connection() as conn:
        # Root ("All Feeds") is treated the same as Uncategorized: a feed placed
        # there is stored folderless (no folder_feeds row) rather than pinned to
        # the root folder, so it surfaces under the virtual Uncategorized folder.
        root_id = get_root_folder_id(conn)
        folderless = folder_id in (UNCATEGORIZED_FOLDER_ID, root_id)
        if folderless:
            conn.execute("DELETE FROM folder_feeds WHERE feed_url = ?", (feed_url,))
        else:
            # Single-folder invariant: a feed lives in exactly one folder. Drop any
            # existing memberships before recording the new one so re-adding a feed
            # moves it rather than leaving it in multiple folders.
            conn.execute(
                "DELETE FROM folder_feeds WHERE feed_url = ? AND folder_id != ?",
                (feed_url, folder_id),
            )
            conn.execute(
                "INSERT OR IGNORE INTO folder_feeds (folder_id, feed_url) VALUES (?, ?)",
                (folder_id, feed_url),
            )
    invalidate_meta_structure_cache()
    if websub_service:
        _uid = tenancy.current_user_id()
        threading.Thread(
            target=websub_service._discover_and_subscribe,
            args=(feed_url, _uid),
            daemon=True,
        ).start()
    # The final stored URL (slash-normalized, or an existing variant reused) —
    # the caller opens this feed, so it must be the key the view resolves.
    return feed_url


# Per-entry meta keyed on (feed_url, entry_id). When an entry moves between
# feeds these have to move with it or the entry arrives stripped: no thumbnail,
# no captured feed tags, and any hand-made correction (title, date, link, body)
# silently reverted. Feed-scoped tables are deliberately absent — they belong to
# the feed being removed, not to its entries.
_ENTRY_META_TABLES = (
    "entry_lead_images",
    "entry_feed_tags",
    "entry_content_edits",
    "entry_content_overrides",
    "entry_date_overrides",
    "entry_link_overrides",
    "entry_title_overrides",
    "entry_media_audio",
    "entry_media_video",
    "entry_read_state",
    "read_history",
    "archived_entries",
)


# Per-entry meta a removed feed leaves behind. Cleaned only for entries with no
# surviving capture — see _purge_dead_entry_meta.
#
# `read_history` is deliberately NOT here. It is a log of what you read, not
# state belonging to a subscription; unsubscribing should not rewrite your
# history. Neither is `saved_entries`, which the star/keep paths own.
_DEAD_ENTRY_META_TABLES = (
    "entry_lead_images",
    "entry_feed_tags",
    "entry_content_edits",
    "entry_content_overrides",
    "entry_date_overrides",
    "entry_link_overrides",
    "entry_title_overrides",
    "entry_media_audio",
    "entry_media_video",
    "entry_read_state",
    "archived_entries",
)


def _purge_dead_entry_meta(conn: sqlite3.Connection, feed_url: str) -> int:
    """Drop per-entry meta for a removed feed's entries that are really gone.

    Feed removal cleaned two marker tables and left everything keyed on
    `(feed_url, entry_id)` behind: 25,504 `entry_lead_images` rows had piled up
    across 371 removed feeds.

    **A capture can outlive its feed**, and that is what makes this a per-entry
    question rather than a `DELETE ... WHERE feed_url = ?`. The Saved view
    renders those archive orphans, and it reads their thumbnail and any
    hand-made corrections from these very tables — 1,923 of the orphaned rows
    were still being displayed. So entries with a surviving `archived_entry` row
    keep their meta; only the rest is dropped.

    Returns the number of rows deleted.
    """
    try:
        with archive_conn() as _ac:
            captured = {str(r[0]) for r in _ac.execute("SELECT entry_id FROM archived_entry WHERE feed_url = ?", (feed_url,))}
    except sqlite3.Error:
        # Cannot prove what is still captured, so delete nothing. Leaking rows
        # is recoverable; blanking a Saved orphan's thumbnail is not.
        LOGGER.warning("[purge] archive unreadable; keeping entry meta for %s", feed_url)
        return 0

    deleted = 0
    if not captured:
        for table in _DEAD_ENTRY_META_TABLES:
            try:
                cur = conn.execute(f"DELETE FROM {table} WHERE feed_url = ?", (feed_url,))
                deleted += cur.rowcount or 0
            except sqlite3.Error:
                LOGGER.debug("[purge] meta cleanup skipped for %s", table, exc_info=True)
        return deleted

    # The captured set goes in a temp table rather than a bound `NOT IN` list.
    # A feed can hold thousands of captures, past SQLite's variable limit, and
    # chunking a NOT IN is actively wrong: each chunk would delete the captures
    # named in every *other* chunk.
    try:
        conn.execute("CREATE TEMP TABLE IF NOT EXISTS _purge_captured (entry_id TEXT PRIMARY KEY)")
        conn.execute("DELETE FROM _purge_captured")
        conn.executemany(
            "INSERT OR IGNORE INTO _purge_captured (entry_id) VALUES (?)",
            [(eid,) for eid in captured],
        )
    except sqlite3.Error:
        LOGGER.warning("[purge] could not stage captured ids; keeping entry meta for %s", feed_url)
        return 0

    for table in _DEAD_ENTRY_META_TABLES:
        try:
            cur = conn.execute(
                f"DELETE FROM {table} WHERE feed_url = ? AND entry_id NOT IN (SELECT entry_id FROM _purge_captured)",
                (feed_url,),
            )
            deleted += cur.rowcount or 0
        except sqlite3.Error:
            LOGGER.debug("[purge] meta cleanup skipped for %s", table, exc_info=True)
    with contextlib.suppress(sqlite3.Error):
        conn.execute("DROP TABLE IF EXISTS _purge_captured")
    return deleted


def _rekey_entry_meta(conn: sqlite3.Connection, from_feed: str, from_id: str, to_feed: str, to_id: str) -> None:
    """Repoint one entry's meta rows at its new (feed_url, entry_id).

    INSERT OR IGNORE then DELETE, per table: the survivor may already have its
    own row (it is the twin the source matched onto), and its own value wins —
    it belongs to the entry that is staying. Best-effort per table so a missing
    one on an older tenant cannot fail a combine.
    """
    if (from_feed, from_id) == (to_feed, to_id):
        return
    for table in _ENTRY_META_TABLES:
        try:
            info = list(conn.execute(f"PRAGMA table_info({table})"))
            # Skip an INTEGER PRIMARY KEY (a rowid alias, e.g. read_history.id):
            # copying it verbatim collides with the row we are copying *from*,
            # INSERT OR IGNORE drops the copy, and the DELETE below then loses
            # the row outright. Omitting it lets SQLite assign a fresh one.
            cols = [r[1] for r in info if not (r[5] and str(r[2] or "").upper() == "INTEGER")]
            if not cols:
                continue
            names = ", ".join(cols)
            picks = ", ".join("?" if c == "feed_url" else ("?" if c == "entry_id" else c) for c in cols)
            params = [to_feed if c == "feed_url" else to_id for c in cols if c in ("feed_url", "entry_id")]
            conn.execute(
                f"INSERT OR IGNORE INTO {table} ({names}) SELECT {picks} FROM {table} WHERE feed_url = ? AND entry_id = ?",
                (*params, from_feed, from_id),
            )
            conn.execute(
                f"DELETE FROM {table} WHERE feed_url = ? AND entry_id = ?",
                (from_feed, from_id),
            )
        except sqlite3.Error:
            LOGGER.debug("[dedup] meta re-key skipped for %s", table, exc_info=True)


def _migrate_curation(reader, conn: sqlite3.Connection, remove_url: str, keep_url: str) -> dict:
    """Move a removed feed's **entries** onto the surviving feed, with their state.

    Each source entry is matched to a survivor entry by GUID, else by normalized
    link; when neither matches it is synthesized into the survivor. Manual tags,
    stars, read/unread state, the offline capture and the per-entry meta rows
    (lead image, feed tags, overrides, history) all follow.

    **Every entry moves, not just curated ones.** This used to walk
    ``set(src_tags) | set(src_stars)`` — an entry with no tag, no star and no
    capture was never visited, so it was neither matched nor synthesized and
    simply vanished when ``reader.delete_feed`` ran. Combining the two Sarah's
    Scribbles Webtoons feeds silently lost an unread post that way. Dropping
    uncurated entries is right for an *unsubscribe*; for a combine — an explicit
    "these two are the same feed" — it is not.

    Read state is carried rather than reset, so combining a large old feed adds
    its history to the survivor without dumping thousands of posts into unread.

    Returns {"tags": n, "stars": n, "synth": n, "archives": n, "entries": n}.
    """
    counts = {"tags": 0, "stars": 0, "synth": 0, "archives": 0, "entries": 0}

    # Which source entries actually hold an offline capture. Read once: the loop
    # below would otherwise open an archive connection per entry just to
    # discover that most have nothing to move.
    try:
        with archive_conn() as _ac:
            src_archived_ids = {str(r[0]) for r in _ac.execute("SELECT entry_id FROM archived_entry WHERE feed_url = ?", (remove_url,))}
    except sqlite3.OperationalError:
        src_archived_ids = set()

    # Ensure the survivor feed exists so synthesized entries have a home.
    try:
        reader.add_feed(keep_url, exist_ok=True)
    except Exception:  # noqa: BLE001
        LOGGER.exception("[dedup] could not ensure survivor feed %s exists", keep_url)
        return counts

    # Index survivor entries: GUID set + normalized-link -> id.
    s_guids: set[str] = set()
    s_link2id: dict[str, str] = {}
    for e in reader.get_entries(feed=keep_url):
        s_guids.add(e.id)
        n = normalize_entry_link_for_dedupe(e.link)
        if n:
            s_link2id.setdefault(n, e.id)

    # Source entries + their manual tags.
    src_entries = {e.id: e for e in reader.get_entries(feed=remove_url)}
    src_tags: dict[str, list[str]] = {}
    for eid, e in src_entries.items():
        keys = [_extract_tag_key(t) for t in reader.get_tags(e.resource_id)]
        keys = [k for k in keys if k and k.startswith(MANUAL_TAG_KEY_PREFIX)]
        if keys:
            src_tags[eid] = keys
    # Source stars (meta saved_entries).
    src_stars: dict[str, str] = {
        r[0]: r[1] for r in conn.execute("SELECT entry_id, saved_at FROM saved_entries WHERE feed_url = ?", (remove_url,))
    }

    # Every source entry, not just the curated ones — see the docstring.
    ids = set(src_entries)
    if not ids and not src_archived_ids:
        return counts

    def _resolve(sid: str) -> tuple[str, bool]:
        if sid in s_guids:
            return sid, False
        e = src_entries.get(sid)
        n = normalize_entry_link_for_dedupe(e.link) if e else None
        if n and n in s_link2id:
            return s_link2id[n], False
        return sid, True  # synth

    for sid in ids:
        target_id, synth = _resolve(sid)
        if synth:
            e = src_entries.get(sid)
            ed: dict = {
                "feed_url": keep_url,
                "id": sid,
                "title": (e.title if e else "") or "",
                "link": (e.link if e and e.link else sid),
            }
            if e and e.published:
                ed["published"] = e.published
            if e and getattr(e, "content", None):
                ed["content"] = [{"value": entry_content.entry_body(e)}]
            elif e and e.summary:
                ed["summary"] = e.summary
            try:
                reader.add_entry(ed)
                counts["synth"] += 1
            except Exception:  # noqa: BLE001
                LOGGER.exception("[dedup] synth add_entry failed %s -> %s", sid, keep_url)
                continue
            target_id = sid
        for key in src_tags.get(sid, []):
            try:
                reader.set_tag((keep_url, target_id), key)
                counts["tags"] += 1
            except Exception:  # noqa: BLE001
                LOGGER.exception("[dedup] set_tag failed %s on %s", key, target_id)
        # Carry the offline capture across too. Without this the archive row
        # stays keyed to a feed that is about to be deleted, and the Saved view
        # then renders it as an archive-only *orphan* — from the archive's own
        # stale link, which is how this surfaced: combined feeds still showing
        # their old, dead URLs. rekey_archive moves the asset links with it and
        # refuses to clobber an existing capture on the target.
        if sid in src_archived_ids:
            try:
                starred_archive_service.rekey_archive(remove_url, sid, keep_url, target_id)
                counts["archives"] += 1
            except Exception:  # noqa: BLE001
                LOGGER.exception("[dedup] archive rekey failed %s -> %s/%s", sid, keep_url, target_id)
        if sid in src_stars:
            cur = conn.execute(
                "INSERT OR IGNORE INTO saved_entries (feed_url, entry_id, saved_at) VALUES (?, ?, ?)",
                (keep_url, target_id, src_stars[sid]),
            )
            if cur.rowcount > 0:  # count real inserts, not IGNOREd existing rows
                counts["stars"] += 1
        # Read state follows the entry. Without this a synthesized entry lands
        # unread by default, so combining an old feed would dump its whole read
        # history into the survivor's unread count. Only ever marks a survivor
        # entry read — a twin already read there is not resurrected as unread by
        # an unread source copy.
        _src = src_entries.get(sid)
        if _src is not None and _src.read:
            try:
                reader.mark_entry_as_read((keep_url, target_id))
                conn.execute(
                    "INSERT INTO entry_read_state (feed_url, entry_id, read_at) VALUES (?, ?, ?)"
                    " ON CONFLICT(feed_url, entry_id) DO UPDATE SET read_at = excluded.read_at",
                    (keep_url, target_id, datetime.now().isoformat()),
                )
            except Exception:  # noqa: BLE001
                LOGGER.exception("[dedup] read-state carry failed %s -> %s", sid, target_id)
        _rekey_entry_meta(conn, remove_url, sid, keep_url, target_id)
        counts["entries"] += 1

    # Archive rows whose entry reader no longer holds — the loop above walks
    # entries, so these are invisible to it. They strand the same way once this
    # feed is deleted. Re-key any whose article exists on the survivor; one with
    # no counterpart there is left alone, since synthesizing an entry purely to
    # host a capture would invent a post that never existed, and the orphan view
    # is already that capture's home.
    for aid in src_archived_ids - ids:
        _target_id, _synth = _resolve(aid)
        if _synth:
            continue
        try:
            starred_archive_service.rekey_archive(remove_url, aid, keep_url, _target_id)
            counts["archives"] += 1
        except Exception:  # noqa: BLE001
            LOGGER.exception("[dedup] archive rekey failed %s -> %s/%s", aid, keep_url, _target_id)

    # The source feed is about to be deleted; drop its now-migrated star rows.
    conn.execute("DELETE FROM saved_entries WHERE feed_url = ?", (remove_url,))
    conn.commit()

    if counts["tags"] or counts["stars"]:
        invalidate_has_manual_tags_cache()
        invalidate_tag_counts_cache()
    return counts


def _move_entry_to_feed(reader, conn: sqlite3.Connection, feed_url: str, entry_id: str, target_url: str) -> dict:
    """Move one entry's curation (star, manual tags, read state) onto *target_url*.

    Single-entry counterpart of ``_migrate_curation``: the entry is matched into
    the target feed by GUID, else normalized link, else synthesized there. The
    source entry itself stays in its feed (reader can't delete feed-provided
    entries) but is marked read and stripped of its star/tags so it stops
    surfacing as curated content.

    Also handles an **orphan-archive source** — a starred entry whose (feed_url,
    entry_id) no longer resolves in reader at all, e.g. the feed was
    unsubscribed, or (reported live 2026-09-14, 142 MakeUseOf stars) its URL
    drifted a trailing slash out from under it after a resubscribe. Sourced from
    the starred archive instead of reader, same as ``_build_orphan_entry_detail``.

    Returns {"ok": bool, "synth": bool, "tags": n, "star": bool, "error": str|None}.
    """
    result = {"ok": False, "synth": False, "tags": 0, "star": False, "content_moved": False, "error": None}
    src = reader.get_entry((feed_url, entry_id), None)
    src_is_orphan = False
    orphan_archived = None
    if src is None:
        orphan_archived = starred_archive_service.get_archived_entry_detail(feed_url, entry_id)
        if orphan_archived is None:
            result["error"] = "Entry not found."
            return result
        src_is_orphan = True
    if orphan_archived is not None:
        src_title = orphan_archived.get("title") or ""
        src_link = html_sanitize.safe_link_url(orphan_archived.get("link")) or entry_id
        src_content_html = orphan_archived.get("content_html") or ""
        src_read = True  # orphan entries carry no reader unread state — always read
    else:
        src_title = src.title or ""
        src_link = src.link or entry_id
        src_content_html = entry_content.entry_body(src)
        src_read = bool(src.read)
    if reader.get_feed(target_url, None) is None:
        result["error"] = "Target feed not found."
        return result
    if target_url == feed_url:
        result["error"] = "Entry is already in that feed."
        return result

    # Resolve the entry inside the target feed: GUID (direct lookup, no feed
    # scan), else normalized link.
    target_id = None
    if reader.get_entry((target_url, entry_id), None) is not None:
        target_id = entry_id
    else:
        src_link_norm = normalize_entry_link_for_dedupe(src_link)
        if src_link_norm:
            for e in reader.get_entries(feed=target_url):
                if normalize_entry_link_for_dedupe(e.link) == src_link_norm:
                    target_id = e.id
                    break
    if target_id is None:
        ed: dict = {
            "feed_url": target_url,
            "id": entry_id,
            "title": src_title,
            "link": src_link,
        }
        # entry_effective_date, not raw src.published: a source with no real
        # publication date (published/updated both absent or garbage) left the
        # synthesized copy with NO published key at all — reader defaults that
        # to epoch-0 (1970-01-01), while its `added` (ingest time) is simply
        # now, since it was just created. The result reads as "brand new" in
        # any date-based view and as "ancient" in any sort keying off the raw
        # column directly, an inconsistency reported live as "confusing as
        # hell" 2026-09-12 — dozens of just-moved GuitarWorld posts sitting at
        # epoch-0 while showing "Now" wherever the effective-date fallback (to
        # `added`) was what actually got displayed. entry_effective_date is
        # the same "published, else updated, else received" chain the list
        # render and mark-older/newer actions already agree on, so the
        # synthesized copy gets a real date under the identical rule instead
        # of silently falling through reader's own default.
        if orphan_archived is not None:
            _published_at = orphan_archived.get("published_at")
            if _published_at:
                try:
                    ed["published"] = datetime.fromtimestamp(float(_published_at), tz=timezone.utc)
                except OverflowError, OSError, ValueError:
                    pass
        else:
            _src_effective_date = entry_effective_date(src)
            if _src_effective_date:
                ed["published"] = _src_effective_date
        if src_content_html:
            ed["content"] = [{"value": src_content_html}]
        try:
            reader.add_entry(ed)
        except Exception:  # noqa: BLE001
            LOGGER.exception("[move-entry] synth add_entry failed %s -> %s", entry_id, target_url)
            result["error"] = "Could not create the entry in the target feed."
            return result
        target_id = entry_id
        result["synth"] = True

    # Carry the body when the target's is thinner. Matching by GUID/link can
    # land on a copy that exists but is *empty* — an auto-filed capture whose
    # source was bot-walled, say — and migrating only curation then moved the
    # star onto a blank entry while the populated source was dropped. That is
    # how a 44KB extension capture of a Medium article became an empty post
    # (reported 2026-07-26). The move must never leave the reader on less than
    # they had; pinned so the target feed's own refresh can't undo it.
    try:
        _src_body = src_content_html
        _tgt = reader.get_entry((target_url, target_id), None)
        _tgt_body = ""
        if _tgt is not None:
            _tgt_body = entry_content.entry_body(_tgt)
        if _src_body and len(_src_body) > len(_tgt_body):
            saved_articles_service.replace_entry_content(
                reader,
                conn,
                target_id,
                "",
                _src_body,
                feed_url=target_url,
                bump_received=False,
                pin_content=True,
            )
            result["content_moved"] = True
    except Exception:  # noqa: BLE001 — curation still migrates; body is best-effort
        LOGGER.exception("[move-entry] could not carry content onto %s/%s", target_url, target_id)

    # Manual tags: re-key onto the target resource, then clear from the source.
    # An orphan source has no reader resource_id — its tags live in
    # orphan_entry_tags instead (see _get_orphan_manual_tags) — but the target
    # is always a real, still-subscribed feed, so the write side is unchanged.
    if src_is_orphan:
        keys = [MANUAL_TAG_KEY_PREFIX + t for t in _get_orphan_manual_tags(feed_url, entry_id)]
    else:
        keys = [_extract_tag_key(t) for t in reader.get_tags(src.resource_id)]
        keys = [k for k in keys if k and k.startswith(MANUAL_TAG_KEY_PREFIX)]
    for key in keys:
        try:
            reader.set_tag((target_url, target_id), key)
            if src_is_orphan:
                conn.execute(
                    "DELETE FROM orphan_entry_tags WHERE feed_url = ? AND entry_id = ? AND tag = ?",
                    (feed_url, entry_id, key[len(MANUAL_TAG_KEY_PREFIX) :]),
                )
            else:
                reader.delete_tag(src.resource_id, key, missing_ok=True)
            result["tags"] += 1
        except Exception:  # noqa: BLE001
            LOGGER.exception("[move-entry] tag move failed %s on %s", key, entry_id)

    # Star: move the saved_entries row.
    row = conn.execute(
        "SELECT saved_at FROM saved_entries WHERE feed_url = ? AND entry_id = ?",
        (feed_url, entry_id),
    ).fetchone()
    if row:
        # OR IGNORE: if the target copy is already starred, its own saved_at
        # wins — the outcome (starred at target, unstarred at source) is the
        # same either way, which is what result["star"] reports.
        conn.execute(
            "INSERT OR IGNORE INTO saved_entries (feed_url, entry_id, saved_at) VALUES (?, ?, ?)",
            (target_url, target_id, row["saved_at"]),
        )
        conn.execute(
            "DELETE FROM saved_entries WHERE feed_url = ? AND entry_id = ?",
            (feed_url, entry_id),
        )
        result["star"] = True

    # Read state: the target copy inherits the source's state; the leftover
    # source entry is marked read so it stops showing up in unread views.
    when = datetime.now().isoformat()
    read_rows = []
    try:
        if src_read:
            reader.mark_entry_as_read((target_url, target_id))
            read_rows.append((target_url, target_id))
        else:
            reader.mark_entry_as_unread((target_url, target_id))
    except Exception:  # noqa: BLE001
        LOGGER.exception("[move-entry] target read-state sync failed %s", target_id)
    if not src_read:
        try:
            reader.mark_entry_as_read((feed_url, entry_id))
            read_rows.append((feed_url, entry_id))
        except Exception:  # noqa: BLE001
            LOGGER.exception("[move-entry] mark source read failed %s", entry_id)
    if read_rows:
        conn.executemany(
            """
            INSERT INTO entry_read_state (feed_url, entry_id, read_at)
            VALUES (?, ?, ?)
            ON CONFLICT(feed_url, entry_id) DO UPDATE SET read_at = excluded.read_at
            """,
            [(fu, eid, when) for fu, eid in read_rows],
        )
    conn.commit()

    # An orphan source has no reader resource to hard-delete — it was never
    # there — so only the archive capture needs re-homing; the source's
    # saved_entries/orphan_entry_tags rows are already cleared above.
    if src_is_orphan:
        try:
            if starred_archive_service.has_complete_archive(target_url, target_id):
                starred_archive_service.delete_archive(feed_url, entry_id)
            else:
                starred_archive_service.rekey_archive(feed_url, entry_id, target_url, target_id)
        except Exception:  # noqa: BLE001 — never fail the move over archive housekeeping
            LOGGER.exception("[move-entry] archive re-key failed for %s", entry_id)
    # A saved article is user-added, so unlike a feed-provided entry it *can* be
    # removed properly — leaving it behind would keep a read, unstarred husk in
    # Saved Articles forever, so the backlog never shrinks as you file it and
    # every later dupe scan re-reads rows that are no longer real saves.
    elif saved_articles_service.is_saved_articles_feed(feed_url):
        # The source archive capture must follow the entry, or it is orphaned:
        # the Read/Saved view renders starred entries from archive rows, so a
        # leftover lectio:saved capture shows as a phantom duplicate of the moved
        # article (with its own, often worse, content). Re-key it onto the
        # target so the capture is preserved; if the target already has one, the
        # source is redundant and re-key deletes it. (This is the recurrence
        # guard for the orphaned-archive-row cleanup in scripts/.)
        try:
            if starred_archive_service.has_complete_archive(target_url, target_id):
                starred_archive_service.delete_archive(feed_url, entry_id)
            else:
                starred_archive_service.rekey_archive(feed_url, entry_id, target_url, target_id)
        except Exception:  # noqa: BLE001 — never fail the move over archive housekeeping
            LOGGER.exception("[move-entry] archive re-key failed for %s", entry_id)
        try:
            _hard_delete_entry(reader, feed_url, entry_id, src)
            result["source_deleted"] = True
        except Exception:  # noqa: BLE001 — the move itself already succeeded
            LOGGER.exception("[move-entry] source cleanup failed for %s", entry_id)
        else:
            # Belt and braces. The star row is moved above, so this should
            # already be a no-op — but a filing run left 3,593 saved_entries
            # rows pointing at entries it had just tombstoned, and that was
            # never reproduced against a copy of the same data. Once the source
            # entry is gone no star row may reference it, so re-issuing the
            # delete is unconditionally correct and closes the hole whatever
            # the cause. Cheap: one indexed delete per filed article.
            try:
                cur = conn.execute(
                    "DELETE FROM saved_entries WHERE feed_url = ? AND entry_id = ?",
                    (feed_url, entry_id),
                )
                conn.commit()
                if cur.rowcount:
                    # Never expected — if this fires, the star row outlived the
                    # move and the root cause is live. Log loudly enough to find.
                    LOGGER.warning(
                        "[move-entry] star row survived the move and was swept: %s %s",
                        feed_url,
                        entry_id,
                    )
            except Exception:  # noqa: BLE001
                LOGGER.exception("[move-entry] star sweep failed for %s", entry_id)

    if result["tags"] or result["star"]:
        invalidate_has_manual_tags_cache()
        invalidate_tag_counts_cache()
    invalidate_unread_counts_cache()
    result["ok"] = True
    return result


def _carry_tags_to_orphan_archive(reader, conn: sqlite3.Connection, feed_url: str) -> int:
    """Copy a feed's manual tags into ``orphan_entry_tags`` before it is deleted.

    Only for entries whose offline capture will outlive the feed — that is the
    same test ``_purge_dead_entry_meta`` uses to decide which per-entry meta
    survives, so the two agree about what still exists afterwards.

    Returns the number of entries whose tags were carried.
    """
    try:
        with archive_conn() as _ac:
            captured = {str(r[0]) for r in _ac.execute("SELECT entry_id FROM archived_entry WHERE feed_url = ?", (feed_url,))}
    except sqlite3.Error:
        # Cannot prove what survives, so carry nothing rather than fabricate
        # tag rows for entries that are about to disappear entirely.
        LOGGER.warning("[purge] archive unreadable; not carrying tags for %s", feed_url)
        return 0
    if not captured:
        return 0

    carried = 0
    for entry in reader.get_entries(feed=feed_url):
        entry_id = str(entry.id)
        if entry_id not in captured:
            continue
        tags = get_manual_tags_for_resource(reader, entry.resource_id)
        if not tags:
            continue
        conn.executemany(
            "INSERT OR IGNORE INTO orphan_entry_tags (feed_url, entry_id, tag) VALUES (?, ?, ?)",
            [(feed_url, entry_id, t) for t in tags],
        )
        carried += 1
    return carried


def purge_orphaned_feed(
    reader,
    conn: sqlite3.Connection,
    feed_url: str,
    *,
    archive_pending: bool = True,
    rescue_to: str | None = None,
    migrate_curation_to: str | None = None,
) -> int:
    """Delete a feed that has already been confirmed orphaned (no folder_feeds rows).

    Must be called AFTER the caller has removed all folder_feeds rows and
    confirmed the feed is no longer referenced.  Does NOT call
    ``invalidate_meta_structure_cache`` — the caller is responsible.

    Steps (in order):
    1. Force-archive any pending saved/starred entries so they don't become
       content-less archive shells after the reader-side data disappears.
       Skipped when *archive_pending* is False (e.g. dedup/upgrade where entries
       survive under the kept/canonical feed).
    2. Rescue unread entries into the kept feed when *rescue_to* is given (dedup
       and format-upgrade paths only).
    3. Delete the feed via the appropriate path:
       - DeviantArt rendered feed → deviantart_service.delete_deviantart_feed
       - dev.to rendered feed     → devto_service.delete_devto_feed
       - Scraped (file://) feed   → scraper_service.delete_scraped_feed
       - Plain feed               → reader.delete_feed
    4. Unsubscribe from the WebSub hub (best-effort; guard for disabled websub).

    Parameters
    ----------
    reader:
        An already-open reader instance (from ``get_reader()``).
    conn:
        An already-open meta-DB connection (from ``get_meta_connection()``).
    feed_url:
        The URL of the feed to remove.
    archive_pending:
        When True (default), force-flush any pending starred-archive captures
        for this feed before deletion.
    rescue_to:
        When set, mark read entries in *rescue_to* as unread when the removed
        feed had them unread (slug-matched).  Used by dedup/upgrade paths.
    migrate_curation_to:
        When set, migrate the removed feed's manual tags and stars onto this
        surviving feed (synthesizing missing entries) before deletion, so
        consolidating duplicates never drops curation.  Used by dedup/upgrade
        paths.

    Returns
    -------
    int
        The number of entries rescued into *rescue_to* (0 when *rescue_to* is
        None).  Dedup callers accumulate this into their JSON ``rescued_count``.
    """
    # Step 1 — force-archive pending saves.
    if archive_pending:
        try:
            forced = starred_archive_service.force_archive_pending_for_feed(feed_url)
            if forced:
                LOGGER.info("[purge] force-archived %d pending captures for %s", forced, feed_url)
        except Exception as exc:  # noqa: BLE001
            LOGGER.warning("[purge] force-archive failed for %s: %s", feed_url, exc)

    # Step 2 — rescue unread entries into the surviving feed.
    rescued = _rescue_unread_entries(reader, feed_url, rescue_to) if rescue_to else 0

    # Step 2b — migrate manual tags + stars onto the surviving feed so a
    # dedup/upgrade consolidation never drops curation.
    if migrate_curation_to:
        migrated = _migrate_curation(reader, conn, feed_url, migrate_curation_to)
        if migrated["tags"] or migrated["stars"] or migrated["synth"] or migrated["archives"]:
            LOGGER.info(
                "[dedup] migrated curation %s -> %s: %d tags, %d stars, %d synthesized, %d archives",
                feed_url,
                migrate_curation_to,
                migrated["tags"],
                migrated["stars"],
                migrated["synth"],
                migrated["archives"],
            )

    # Step 2c — carry manual tags across to the orphan-archive side.
    #
    # A tag is a keep signal exactly like a star, but the two are stored in very
    # different places: a star lives in `saved_entries` (meta DB, no reader
    # dependency) and survives the feed's deletion untouched, while a tag lives
    # in reader's own entry_tags and is deleted WITH the feed. So a tagged-but-
    # unstarred entry used to come out of an unsubscribe with its capture intact
    # and its tags gone.
    #
    # That is worse than untidy. An orphan archive counts as kept only if it is
    # starred OR manually tagged (see "A surviving capture is not itself a keep
    # signal"), so losing the tag makes the entry read as carrying no keep signal
    # at all — unreachable in Saved, and eligible for deletion by
    # scripts/purge_uncurated_orphan_archives.py. Tagging something is supposed
    # to keep it forever; this is what makes that true across an unsubscribe.
    #
    # orphan_entry_tags is the same table the UI already writes when you tag an
    # entry whose feed is gone, so nothing new is invented here — the tags simply
    # move to where an orphan's tags are meant to live.
    #
    # Gated on a surviving capture, matching _purge_dead_entry_meta's rule: with
    # no archive row the entry is entirely gone, and a tag pointing at nothing
    # would be a ghost. Skipped when migrate_curation_to is set, because there
    # the tags follow the entries onto the surviving feed instead.
    if not migrate_curation_to:
        _carried = _carry_tags_to_orphan_archive(reader, conn, feed_url)
        if _carried:
            LOGGER.info("[purge] carried tags for %d orphan capture(s) on %s", _carried, feed_url)

    # Step 3 — dispatch the delete via the appropriate path.
    da_id = deviantart_service.deviantart_feed_id_from_url(feed_url)
    devto_id = None if da_id else devto_service.devto_feed_id_from_url(feed_url)
    feed_id = None if (da_id or devto_id) else scraper_service.scraped_feed_id_from_url(feed_url)
    if da_id:
        deviantart_service.delete_deviantart_feed(conn, reader, da_id)
    elif devto_id:
        devto_service.delete_devto_feed(conn, reader, devto_id)
    elif feed_id:
        scraper_service.delete_scraped_feed(conn, reader, feed_id)
    else:
        reader.delete_feed(feed_url, missing_ok=True)

    # Prune captured feed-provided tags (best-effort).
    try:
        feed_tag_service.delete_for_feed(feed_url)
    except Exception:  # noqa: BLE001
        LOGGER.warning("[purge] entry_feed_tags cleanup failed for %s", feed_url)

    # Drop any kept-feed / needs-replacement markers for a feed being removed,
    # and its folder membership.
    #
    # The folder row is the one users actually see leak. Removing a feed deletes
    # it from reader but left `folder_feeds` pointing at it, so Settings → Feeds
    # went on listing a subscription that no longer exists — rendered from the
    # folder row alone, with a failing badge, indistinguishable from a real dead
    # feed. The 29 article-URL husks rehomed to Saved Articles on 2026-08-06 all
    # came back as ghosts this way and were reported as "non-feeds I thought we
    # cleared out". Callers that manage folders themselves (the dedupe route,
    # `_migrate_curation`'s combine) already delete the row first; this is
    # idempotent, and it is the backstop for every caller that does not.
    #
    # Safe unconditionally: an unsubscribe-with-keep is defined as leaving the
    # tree — its curated items stay reachable through the Kept view, which does
    # not read folder_feeds.
    # feed_failure_state is here for the same reason, one layer down: it is the
    # record of a feed's consecutive failures, and nothing ever cleared it on
    # removal. A feed unsubscribed *because* it was dead therefore stayed in the
    # failure tables forever, so the Failing Feeds counts and the "dead — needs
    # replacement" triage kept reporting feeds that no longer exist — the 404
    # sweep on 2026-08-11/12 left 560 such rows behind. It is pure derived state
    # rebuilt on the next fetch, so dropping it cannot lose anything: a feed that
    # is re-added starts from a clean slate, which is the correct reading of a
    # deliberate re-subscribe.
    try:
        conn.execute("DELETE FROM kept_feeds WHERE feed_url = ?", (feed_url,))
        conn.execute("DELETE FROM feeds_needing_replacement WHERE feed_url = ?", (feed_url,))
        conn.execute("DELETE FROM folder_feeds WHERE feed_url = ?", (feed_url,))
        conn.execute("DELETE FROM feed_failure_state WHERE feed_url = ?", (feed_url,))
    except Exception:  # noqa: BLE001
        LOGGER.warning("[purge] kept_feeds/needs_replacement/folder/failure cleanup failed for %s", feed_url)

    _dead_meta = _purge_dead_entry_meta(conn, feed_url)
    if _dead_meta:
        LOGGER.info("[purge] dropped %d dead per-entry meta row(s) for %s", _dead_meta, feed_url)

    # Step 4 — WebSub unsubscribe (best-effort; websub_service may be None).
    if websub_service:
        websub_service.unsubscribe(feed_url, tenancy.current_user_id())

    return rescued


def remove_feed_from_folder(feed_url: str, folder_id: int) -> None:
    """Remove a feed from a folder, and delete it from reader if it's in no other folder."""
    feed_url = feed_url.strip()
    if not feed_url:
        return
    with get_meta_connection() as conn:
        conn.execute(
            "DELETE FROM folder_feeds WHERE folder_id = ? AND feed_url = ?",
            (folder_id, feed_url),
        )
        still_exists = conn.execute(
            "SELECT 1 FROM folder_feeds WHERE feed_url = ? LIMIT 1",
            (feed_url,),
        ).fetchone()
    if not still_exists:
        with get_reader() as reader:
            with get_meta_connection() as conn:
                purge_orphaned_feed(reader, conn, feed_url, archive_pending=True)
    invalidate_meta_structure_cache()


def get_push_active_feed_urls() -> set[str]:
    """Return the set of feed URLs with a verified, active WebSub push subscription.

    Queries the shared (non-per-user) websub DB. Returns an empty set when
    WebSub is disabled (LECTIO_PUBLIC_URL not set) or the table is empty.
    Only feeds with verified=1 AND hub_url IS NOT NULL are included.
    """
    if not websub_service:
        return set()
    try:
        conn = get_websub_connection()
        rows = conn.execute("SELECT feed_url FROM websub_subscriptions WHERE verified=1 AND hub_url IS NOT NULL").fetchall()
        return {str(row["feed_url"]) for row in rows}
    except Exception:
        LOGGER.debug("[websub] could not load push-active feed URLs", exc_info=True)
        return set()


@busy_work("YouTube sync")
def _run_youtube_sync(folder_id: int | None = None) -> dict:
    """Run YouTube subscription sync, creating the target folder if needed.

    If folder_id is None, looks up or creates yt_folder_name.
    Returns the result dict from sync_youtube_folder.
    """
    yt_api_key = get_yt_api_key()
    yt_channel_id = get_yt_channel_id()
    yt_folder_name = get_yt_folder_name()
    if not yt_api_key:
        return {"added": 0, "removed": 0, "total": 0, "error": "YouTube API key is not configured."}
    if not yt_channel_id:
        return {"added": 0, "removed": 0, "total": 0, "error": "YouTube channel ID is not configured."}

    # Resolve or create the target folder.
    if folder_id is None:
        with get_meta_connection() as conn:
            row = conn.execute(
                "SELECT id FROM folders WHERE name = ? LIMIT 1",
                (yt_folder_name,),
            ).fetchone()
            if row:
                folder_id = int(row["id"])
            else:
                root_id = get_root_folder_id(conn)
                conn.execute(
                    "INSERT INTO folders (name, parent_id) VALUES (?, ?)",
                    (yt_folder_name, root_id),
                )
                folder_id = conn.execute("SELECT last_insert_rowid()").fetchone()[0]
        invalidate_meta_structure_cache()

    def _get_folder_feed_urls(fid: int) -> list[str]:
        with get_meta_connection() as conn:
            rows = conn.execute(
                "SELECT feed_url FROM folder_feeds WHERE folder_id = ?",
                (fid,),
            ).fetchall()
        return [str(r["feed_url"]) for r in rows]

    result = sync_youtube_folder(
        api_key=yt_api_key,
        channel_identifier=yt_channel_id,
        folder_id=folder_id,
        get_folder_feed_urls=_get_folder_feed_urls,
        add_feed=add_feed_to_folder,
        remove_feed=remove_feed_from_folder,
    )

    # Record last-sync time and summary for display in the UI. Use time.strftime
    # (local time with a real %Z) — datetime.now() is naive, so its %Z renders
    # empty and the timestamp shows without a zone unlike the other stats.
    now_iso = time.strftime("%Y-%m-%d %H:%M %Z")
    if result["error"]:
        last_result = f"Error: {result['error']}"
    else:
        last_result = f"+{result['added']} / -{result['removed']} ({result['total']} subs)"
    with get_meta_connection() as conn:
        set_setting(conn, YOUTUBE_SYNC_LAST_AT_KEY, now_iso)
        set_setting(conn, YOUTUBE_SYNC_LAST_RESULT_KEY, last_result)

    return result


def move_feed_to_folder(feed_url: str, from_folder_id: int, to_folder_id: int) -> None:
    feed_url = feed_url.strip()
    if not feed_url:
        raise ValueError("Feed URL is required.")
    if from_folder_id == to_folder_id:
        return

    with get_meta_connection() as conn:
        # Root ("All Feeds") is treated the same as Uncategorized: moving a feed
        # there leaves it folderless (no folder_feeds row) so it surfaces under
        # the virtual Uncategorized folder rather than being pinned to root.
        root_id = get_root_folder_id(conn)
        folderless = to_folder_id in (UNCATEGORIZED_FOLDER_ID, root_id)
        if not folderless:
            target_row = conn.execute(
                "SELECT id FROM folders WHERE id = ?",
                (to_folder_id,),
            ).fetchone()
            if not target_row:
                raise ValueError("Target folder does not exist.")

        if folderless:
            conn.execute(
                "DELETE FROM folder_feeds WHERE feed_url = ?",
                (feed_url,),
            )
        else:
            conn.execute(
                "DELETE FROM folder_feeds WHERE folder_id = ? AND feed_url = ?",
                (from_folder_id, feed_url),
            )
            conn.execute(
                "INSERT OR IGNORE INTO folder_feeds (folder_id, feed_url) VALUES (?, ?)",
                (to_folder_id, feed_url),
            )
    invalidate_meta_structure_cache()


def delete_folder(
    folder_id: int,
    feed_action: str = "unsub",
    move_to_folder_id: int | None = None,
) -> tuple[int, int, int]:
    """Delete a folder (and its descendants).

    feed_action controls what happens to feeds that live in the deleted folders:
      - "unsub" (default): unsubscribe feeds that end up in no other folder.
      - "move": reassign every affected feed to move_to_folder_id. A target of
        UNCATEGORIZED_FOLDER_ID (or the root folder) leaves feeds folderless
        (i.e. Uncategorized) without unsubscribing them.

    Returns (deleted_folder_count, unsubscribed_feed_count, moved_feed_count).
    """
    if feed_action not in ("unsub", "move"):
        raise ValueError("Invalid feed action.")

    with get_meta_connection() as conn:
        root_id = get_root_folder_id(conn)
        row = conn.execute(
            "SELECT id, parent_id FROM folders WHERE id = ?",
            (folder_id,),
        ).fetchone()
        if not row:
            return (0, 0, 0)
        if int(row["id"]) == root_id:
            raise ValueError("Cannot delete root folder.")

        descendant_ids = get_descendant_folder_ids(conn, folder_id)
        placeholders = ",".join("?" for _ in descendant_ids)

        target_id: int | None = None
        if feed_action == "move":
            if move_to_folder_id is None:
                raise ValueError("A target folder is required to move feeds.")
            target_id = int(move_to_folder_id)
            # Treat the root folder the same as Uncategorized: feeds become folderless.
            if target_id == root_id:
                target_id = UNCATEGORIZED_FOLDER_ID
            if target_id in descendant_ids:
                raise ValueError("Cannot move feeds into the folder being deleted.")
            if target_id != UNCATEGORIZED_FOLDER_ID:
                target_row = conn.execute(
                    "SELECT id FROM folders WHERE id = ?",
                    (target_id,),
                ).fetchone()
                if not target_row:
                    raise ValueError("Target folder does not exist.")

        feed_rows = conn.execute(
            f"SELECT DISTINCT feed_url FROM folder_feeds WHERE folder_id IN ({placeholders})",
            descendant_ids,
        ).fetchall()
        affected_feed_urls = [str(r["feed_url"]) for r in feed_rows]

        conn.execute(
            f"DELETE FROM folder_feeds WHERE folder_id IN ({placeholders})",
            descendant_ids,
        )
        conn.execute(
            f"DELETE FROM folders WHERE id IN ({placeholders})",
            descendant_ids,
        )

        moved_feed_count = 0
        if feed_action == "move":
            if target_id != UNCATEGORIZED_FOLDER_ID:
                for feed_url in affected_feed_urls:
                    conn.execute(
                        "INSERT OR IGNORE INTO folder_feeds (folder_id, feed_url) VALUES (?, ?)",
                        (target_id, feed_url),
                    )
            moved_feed_count = len(affected_feed_urls)
            invalidate_meta_structure_cache()
            return (len(descendant_ids), 0, moved_feed_count)

        orphaned_feed_urls: list[str] = []
        for feed_url in affected_feed_urls:
            still_used = conn.execute(
                "SELECT 1 FROM folder_feeds WHERE feed_url = ? LIMIT 1",
                (feed_url,),
            ).fetchone()
            if not still_used:
                orphaned_feed_urls.append(feed_url)

    removed_feed_count = 0
    if orphaned_feed_urls:
        with get_reader() as reader:
            with get_meta_connection() as conn:
                for feed_url in orphaned_feed_urls:
                    try:
                        purge_orphaned_feed(reader, conn, feed_url, archive_pending=True)
                    except Exception:
                        continue
                    conn.execute(
                        "INSERT OR REPLACE INTO declined_feeds (feed_url, declined_at) VALUES (?, ?)",
                        (feed_url, datetime.now().isoformat()),
                    )
                    removed_feed_count += 1
                conn.commit()

    invalidate_meta_structure_cache()
    return (len(descendant_ids), removed_feed_count, 0)


def _start_background_update(feed_url: str) -> None:
    with get_meta_connection() as conn:
        if feed_url in get_disabled_feed_urls(conn):
            return
    _uid = tenancy.current_user_id()  # captured here; the thread cannot see it

    def _run() -> None:
        with updating_feeds_lock:
            if feed_url in updating_feeds:
                return
            updating_feeds.add(feed_url)
        try:
            feed_refresh_service.update_feeds([feed_url])
            _run_automation_after_refresh({feed_url})
        finally:
            with updating_feeds_lock:
                updating_feeds.discard(feed_url)

    thread = threading.Thread(target=_run_in_user_context, args=(_uid, _run), daemon=True)
    thread.start()


_FOLDER_CADENCE_LAST_REFRESH_PREFIX = "folder_cadence_last_refresh:"
# Feeds in no folder ("Uncategorized") are invisible to the per-folder loop, so
# they get their own attempt-clock at the global cadence. Without this a feed
# added to Uncategorized never refreshes at all — no folder ever selects it.
_UNCATEGORIZED_CADENCE_LAST_REFRESH_KEY = "uncategorized_cadence_last_refresh"


def _background_user_ids() -> list[str]:
    """Users that background work (scheduled refresh, maintenance) should run for.

    Returns every enabled account; each is processed under its own tenancy context."""
    return [u["user_id"] for u in user_store.list_users() if not u["disabled"]]


def _for_each_background_user(label: str, fn: Callable[[], None]) -> None:
    """Run ``fn()`` once per background user, each under its own tenancy context.

    Startup backfills, syncs and one-off cleanups touch per-user DBs through the
    context-bound ``get_reader()`` / ``get_meta_connection()`` helpers. Run bare,
    they raise ``TenancyUnboundError``. Wrapping the call here binds each enabled user in turn.
    One user's failure is logged and does not abort the rest."""
    for uid in _background_user_ids():
        with tenancy.user_context(uid):
            try:
                fn()
            except Exception:
                LOGGER.exception("[startup] %s failed for user %r", label, uid)


def _effective_auto_refresh_minutes() -> int:
    """Auto-refresh cadence for the currently-bound user."""
    with get_meta_connection() as conn:
        return get_auto_refresh_minutes(conn)


def _rotate_for_fairness(uids: list[str]) -> list[str]:
    """Rotate the per-tick user order round-robin so the same user isn't always
    processed first.

    Users are refreshed sequentially within a tick (adequate at the 1–3 trusted
    user target — every user is still processed every tick). Rotating the start
    point each pass spreads any first-mover advantage and means a slow or hanging
    user delays a different set of downstream users each time rather than always
    the same ones. Deeper fairness at scale (per-user concurrency, fetch budgets)
    stays deferred behind this seam per the multi-user plan."""
    if len(uids) <= 1:
        return uids
    offset = next_refresh_rotation_offset(len(uids))
    return uids[offset:] + uids[:offset]


def _run_scheduled_refresh_for_all_users() -> None:
    """One scheduled-refresh pass across every background user, each under its
    own tenancy context so the refresh hits that user's databases."""
    now = time.monotonic()
    with _scheduler_state_lock:
        _scheduler_state["pass_started_at"] = now
        _scheduler_state["last_progress_at"] = now
        _scheduler_state["stage"] = "starting"
        _scheduler_state["consecutive_stall_logs"] = 0
    try:
        for uid in _rotate_for_fairness(_background_user_ids()):
            _note_scheduler_progress(f"user {uid}")
            with tenancy.user_context(uid):
                try:
                    _scheduled_refresh_tick()
                except Exception:
                    LOGGER.exception("scheduled refresh failed for user %r", uid)
        # Renewal is global (shared DB) — run once per scheduler tick, not per user.
        # Guarded: it sits outside the per-user loop, so an exception here escaped
        # into scheduled_refresh_loop and killed the scheduler thread permanently —
        # the same 34-hour silence as a hung socket, one `try` away.
        if websub_service:
            _note_scheduler_progress("websub renewal")
            try:
                websub_service.renew_expiring_subscriptions()
            except Exception:
                LOGGER.exception("[websub] subscription renewal failed")
    finally:
        with _scheduler_state_lock:
            _scheduler_state["pass_started_at"] = None
            _scheduler_state["last_pass_finished_at"] = time.monotonic()
            _scheduler_state["stage"] = "idle"


def scheduled_refresh_loop(stop_event: threading.Event) -> None:
    while not stop_event.wait(SCHEDULER_POLL_SECONDS):
        # Nothing may escape this loop. An uncaught exception here does not crash
        # the app — it silently ends the only thread that refreshes feeds, while
        # /healthz stays green and the UI looks fine.
        try:
            _run_scheduled_refresh_for_all_users()
        except Exception:
            LOGGER.exception("scheduled refresh pass failed; the loop continues")


def scheduler_watchdog_loop(stop_event: threading.Event) -> None:
    """Watch the scheduler for a pass that has stopped advancing.

    The failure this exists for is not that refresh broke — it is that refresh
    broke *invisibly*. The scheduler thread stayed alive, blocked on a socket read
    with no deadline, and nothing refreshed for 34 hours while /healthz answered
    200 the whole time.

    Progress, not elapsed time, is the signal: a full-library pass legitimately
    runs for an hour, so "this pass is taking a long time" says nothing. A pass
    that has not advanced by even one feed for ten minutes is stuck.

    A stuck pass cannot be cancelled — Python has no way to interrupt a thread
    blocked in a socket read — so the escalation is to exit the process and let
    the container's restart policy bring it back, which is exactly the manual
    recovery that worked. Set LECTIO_SCHEDULER_STALL_RESTART_SECONDS=0 to log only.
    """
    while not stop_event.wait(SCHEDULER_WATCHDOG_POLL_SECONDS):
        try:
            stalled_for = _scheduler_stall_seconds()
            if stalled_for is None or stalled_for < SCHEDULER_STALL_SECONDS:
                continue
            with _scheduler_state_lock:
                stage = str(_scheduler_state.get("stage") or "?")
                logs = cast(int, _scheduler_state.get("consecutive_stall_logs") or 0)
                _scheduler_state["consecutive_stall_logs"] = logs + 1
            LOGGER.error(
                "[scheduler] STALLED: no progress for %ds while doing %r. Feeds are not refreshing. Restart threshold: %s",
                int(stalled_for),
                stage,
                f"{SCHEDULER_STALL_RESTART_SECONDS}s" if SCHEDULER_STALL_RESTART_SECONDS > 0 else "disabled",
            )
            if 0 < SCHEDULER_STALL_RESTART_SECONDS <= stalled_for:
                LOGGER.error(
                    "[scheduler] stalled %ds at %r — exiting so the container restarts",
                    int(stalled_for),
                    stage,
                )
                # os._exit, not sys.exit: the point is that a thread is wedged in
                # a syscall, so an orderly shutdown would block on joining it.
                logging.shutdown()
                os._exit(1)
        except Exception:
            LOGGER.exception("[scheduler] watchdog iteration failed")


@busy_work("scheduled refresh")
def _scheduled_refresh_tick() -> None:
    """One scheduled-refresh pass for the currently-bound tenancy user."""
    global_minutes = _effective_auto_refresh_minutes()
    if global_minutes <= 0:
        return

    now_ts = time.time()
    feeds_to_refresh: set[str] = set()
    folders_to_mark: list[tuple[str, str]] = []

    # Feeds paused via Feed Properties set reader's updates_enabled=0. The scheduler
    # drives per-feed reader.update_feed(), which ignores that flag, so we must
    # exclude paused feeds here ourselves (alongside Lectio's own disabled_feeds).
    with get_reader() as reader:
        paused = {str(f.url) for f in reader.get_feeds(updates_enabled=False)}
        enabled_feed_urls = {str(f.url) for f in reader.get_feeds(updates_enabled=True)}
    with get_meta_connection() as conn:
        disabled = get_disabled_feed_urls(conn) | paused
        # Load all folders and their per-folder cadence settings.
        folder_rows = conn.execute("SELECT id, cadence_minutes FROM folders").fetchall()
        for folder in folder_rows:
            fid = int(folder["id"])
            cadence = folder["cadence_minutes"]
            effective_minutes = cadence if (cadence and cadence > 0) else global_minutes
            key = f"{_FOLDER_CADENCE_LAST_REFRESH_PREFIX}{fid}"
            last_ts_str = get_setting(conn, key)
            last_ts = float(last_ts_str) if last_ts_str else 0.0
            if (now_ts - last_ts) < effective_minutes * 60:
                continue
            # This folder is due — collect its feeds.
            folder_feed_rows = conn.execute("SELECT feed_url FROM folder_feeds WHERE folder_id = ?", (fid,)).fetchall()
            for row in folder_feed_rows:
                url = str(row["feed_url"])
                if url not in disabled:
                    feeds_to_refresh.add(url)
            folders_to_mark.append((key, str(now_ts)))

        # Uncategorized feeds: those in no folder never appear above, so refresh
        # them as one bucket on the global cadence. update_feeds still applies
        # its own per-feed/domain/429 backoff, so this only decides candidacy.
        foldered = {str(row["feed_url"]) for row in conn.execute("SELECT DISTINCT feed_url FROM folder_feeds")}
        orphan_feeds = {url for url in enabled_feed_urls if url not in foldered and url not in disabled}
        if orphan_feeds:
            last_ts_str = get_setting(conn, _UNCATEGORIZED_CADENCE_LAST_REFRESH_KEY)
            last_ts = float(last_ts_str) if last_ts_str else 0.0
            if (now_ts - last_ts) >= global_minutes * 60:
                feeds_to_refresh |= orphan_feeds
                folders_to_mark.append((_UNCATEGORIZED_CADENCE_LAST_REFRESH_KEY, str(now_ts)))

        if folders_to_mark:
            for key, val in folders_to_mark:
                set_setting(conn, key, val)
            # set_setting opened an implicit transaction on this pooled connection; the refresh rounds below share it and fetch over
            # the network, so commit first or the first fetch of every round holds the write lock.
            conn.commit()
            # Keep legacy global timestamp for monitoring / debug.
            app.state.last_scheduled_refresh_started_at = time.monotonic()
            with get_meta_connection() as _sc:
                scraper_service.refresh_all_scraped_feeds(_sc)
                devto_service.refresh_all_devto_feeds(_sc)
                _da_cid, _da_secret = get_deviantart_credentials()
                deviantart_service.refresh_all_deviantart_feeds(_sc, _da_cid, _da_secret, access_token=get_deviantart_user_token())
                _reddit_token = get_reddit_user_token()
                if _reddit_token:
                    try:
                        with get_reader() as _rdr:
                            reddit_service.refresh_all_reddit_feeds(_sc, _rdr, _reddit_token)
                    except Exception:
                        LOGGER.exception("[reddit] feed sync error in scheduler")
            if inoreader_connected():
                try:
                    _inoreader_drip_step()
                except Exception:
                    LOGGER.exception("[inoreader] drip step error in scheduler")

    if not feeds_to_refresh:
        return

    if REFRESH_DEBUG_ENABLED:
        LOGGER.info(
            "[refresh] scheduled run: global_minutes=%d feed_count=%d folder_count=%d",
            global_minutes,
            len(feeds_to_refresh),
            len(folders_to_mark),
        )
    # Refresh without enhancement so automation (hide-shorts, mark-read, dedup)
    # runs as soon as entries land, instead of queuing behind the network-heavy
    # lead-image / duration pass over the whole batch.
    feed_refresh_service.update_feeds(feeds_to_refresh, enhance=False, inline_followups=True)
    _run_automation_after_refresh(feeds_to_refresh)
    invalidate_unread_counts_cache()
    if websub_service:
        websub_service.maybe_discover_hubs(list(feeds_to_refresh), tenancy.current_user_id())
    # Enhancement still runs in this scheduler thread (same total work as before),
    # just after the fast automation pass. A second hide-shorts pass afterward
    # catches ≤60s videos that only the freshly-fetched durations can identify;
    # invalidate again only if it actually marked something.
    _enhance_feeds_background(list(feeds_to_refresh))
    if _apply_hide_shorts(feeds_to_refresh):
        invalidate_unread_counts_cache()


def _run_daily_maintenance() -> None:
    """Nightly cleanup. Per-user work (log prune, orphan cleanup, VACUUM of the
    user's meta/starred DBs, email-batch flush) runs once per enabled user under
    their tenancy context; global work (thumb-cache VACUUM, YouTube sync) once."""
    LOGGER.info("[maintenance] starting daily maintenance")
    for _uid in _background_user_ids():
        with tenancy.user_context(_uid):
            try:
                _daily_maintenance_for_user()
            except Exception:
                LOGGER.exception("[maintenance] failed for user %r", _uid)
    _run_global_maintenance()
    LOGGER.info("[maintenance] daily maintenance complete")


# NOT wired into nightly maintenance, deliberately. Measured 2026-07-26: a
# freshly-signed mature-deviation URL expires in about **15 minutes**, not the
# week the first sample suggested — so a 3am re-sign hands back images that are
# dead by 3:15. Re-signing has to happen when the post is actually opened; this
# routine stays as the building block for that (and for a manual catch-up via
# scripts/refresh_expired_deviantart_images.py, which does make the image work
# right now).
# Bound one run's API calls. Mature deviations are a small slice of a library
# and each needs its own request; anything beyond this waits for tomorrow rather
# than hammering DeviantArt in one burst.
_DA_IMAGE_REFRESH_MAX_PER_RUN = 100


def refresh_expiring_deviantart_images(
    *,
    within_seconds: float = 0.0,
    max_entries: int = _DA_IMAGE_REFRESH_MAX_PER_RUN,
    apply: bool = True,
) -> dict:
    """Re-sign stored DeviantArt image URLs whose token has expired or is close to.

    Matches on the *stored HTML* rather than a maturity flag: is_mature isn't
    recorded, and the token is the thing that actually breaks. Runs for the
    currently-bound tenancy user. Returns {"stale", "refreshed"}.
    """
    cutoff = time.time() + max(0.0, within_seconds)
    stale: list[tuple[str, str, str]] = []  # (feed, entry_id, column)
    with get_reader() as reader:
        rows = (
            reader._storage.get_db()
            .execute(
                "SELECT feed, id, summary, content FROM entries"
                " WHERE COALESCE(summary, '') LIKE '%wixmp%token=%'"
                "    OR COALESCE(content, '') LIKE '%wixmp%token=%'"
            )
            .fetchall()
        )
    for feed, entry_id, summary, content in rows:
        for column, body in (("summary", summary), ("content", content)):
            if not body or "wixmp" not in body:
                continue
            exp = deviantart_service.image_token_expiry(body)
            if exp is not None and exp < cutoff:
                stale.append((str(feed), str(entry_id), column))
                break
    if not apply or not stale:
        return {"stale": len(stale), "refreshed": 0}

    # Not the raw stored value: DA access tokens last an hour, so a nightly job
    # reading app_settings directly would 401 on almost every run. This is the
    # same accessor the watch-list sync uses, refreshing from the refresh token
    # when needed and returning "" when the session is dead.
    token = get_deviantart_user_token()
    if not token:
        LOGGER.info(
            "[deviantart] %d image(s) need re-signing but no usable access token (not connected, or reconnect required)", len(stale)
        )
        return {"stale": len(stale), "refreshed": 0}

    refreshed = 0
    with get_reader() as reader:
        db = reader._storage.get_db()
        for feed, entry_id, column in stale[:max_entries]:
            # For DeviantArt feeds the stored entry id *is* the deviation id.
            fresh = deviantart_service.fetch_fresh_image_url(entry_id, token)
            if not fresh:
                continue
            body_row = db.execute(
                f"SELECT {column} FROM entries WHERE feed = ? AND id = ?",  # nosemgrep: column is a literal
                (feed, entry_id),
            ).fetchone()
            if not body_row or not body_row[0]:
                continue
            old = re.search(r'src="([^"]*wixmp[^"]*)"', body_row[0])
            if not old:
                continue
            db.execute(
                f"UPDATE entries SET {column} = ? WHERE feed = ? AND id = ?",  # nosemgrep: column is a literal
                (body_row[0].replace(old.group(1), fresh.replace("&", "&amp;")), feed, entry_id),
            )
            refreshed += 1
            time.sleep(0.3)  # stay a polite API client
        db.commit()
    return {"stale": len(stale), "refreshed": refreshed}


# Capture the whole src value in one linear group and test for "wixmp" in
# Python. Embedding the literal between two unbounded [^"]* runs backtracks
# polynomially on feed-supplied HTML with many repetitions and no closing quote
# (CodeQL: polynomial regular expression used on uncontrolled data).
_IMG_SRC_ATTR_RE = re.compile(r'(<img\b[^>]*?\bsrc\s*=\s*")([^"]*)(")', re.IGNORECASE)


def _img_cache_has(url: str) -> bool:
    """Is this image already in the proxy's byte cache? The cache key drops the
    signing token (_img_cache_key_url), so a copy fetched under one token still
    answers for every later one — which is what makes an expired DeviantArt URL
    keep rendering."""
    try:
        key = hashlib.sha256(_img_cache_key_url(url).encode("utf-8")).hexdigest()
        with get_img_cache_connection() as conn:
            return conn.execute("SELECT 1 FROM img_cache WHERE cache_key = ? LIMIT 1", (key,)).fetchone() is not None
    except Exception:  # noqa: BLE001 — a cache miss is the safe answer
        return False


def _wixmp_url_is_live(url: str) -> bool:
    """Cheap liveness check for a wixmp URL whose token carries no readable
    ``exp`` claim. That does NOT mean permanent — a GIF whose JWT had no exp
    at all turned out to already be dead (wixmp answered 400 "image is
    invalid" directly) — so a token with no way to prove itself fresh gets one
    direct HEAD before being trusted. Same SSRF guard as any other outbound
    fetch; failures (including a network error) are treated as "not live" so
    the caller falls through to a real re-sign rather than trusting a URL that
    could not be checked."""
    try:
        if not url_guard.is_safe_outbound_url(url):
            return False
        with url_guard.build_client(timeout=5.0, headers={"User-Agent": READABILITY_USER_AGENT}) as client:
            resp = client.head(url)
        return resp.status_code == 200 and resp.headers.get("content-type", "").startswith("image/")
    except Exception:
        return False


def _resign_expired_deviantart_url(url: str | None, entry_id: str) -> str | None:
    """Re-sign one dead DeviantArt image URL, or return it unchanged.

    The single place that decides *whether* a re-sign is warranted, so the body
    HTML and the lead image (resolved on a different path, from the lead-image
    cache) can't drift apart — they did: the body was re-signed while the lead
    image, which is what the article actually displays, kept its dead token.
    """
    if not url or "wixmp" not in url:
        return url
    if _img_cache_has(url):
        return url  # the proxy holds the bytes; the token no longer matters
    exp = deviantart_service.image_token_expiry(url)
    if exp is not None and exp > time.time():
        return url  # still valid
    if exp is None and _wixmp_url_is_live(url):
        return url  # no exp claim, and it just proved itself live
    token = get_deviantart_user_token()
    if not token:
        return url
    return deviantart_service.fetch_fresh_image_url(entry_id, token) or url


def _resign_expired_deviantart_images(content_html: str, feed_url: str, entry_id: str) -> str:
    """Re-sign a DeviantArt image whose URL token has already expired.

    Mature deviations are signed for only ~15 minutes, so a stored URL is dead
    long before it is read and the post loses both image and thumbnail. A
    scheduled re-sign can't help at that lifetime — the moment that matters is
    when the post is opened, which is here.

    Three things keep this cheap. The proxy's byte cache is consulted first: once
    an image has been fetched under *any* valid token it answers forever (the
    cache key drops the token), so a re-sign happens at most once per image
    rather than once per view. A still-valid (readable ``exp``, not yet passed)
    token is left alone. And a token with no ``exp`` claim at all — the vast
    majority, since ordinary deviations sign that way — gets one direct HEAD
    at the image host (_wixmp_url_is_live) rather than a DeviantArt API call;
    only a URL that fails even that reaches the API. No-``exp`` does NOT mean
    permanent — one turned out to already be dead with nothing in the token to
    say so (2026-08-20) — so this only skips the *paid* API call for the
    common case, not the correctness check itself.
    """
    if not content_html or "wixmp" not in content_html:
        return content_html
    replaced: dict[str, str] = {}

    def _sub(m: re.Match) -> str:
        url = html.unescape(m.group(2))
        if "wixmp" not in url:
            return m.group(0)
        if url in replaced:
            return f"{m.group(1)}{html.escape(replaced[url], quote=True)}{m.group(3)}"
        fresh = _resign_expired_deviantart_url(url, entry_id)
        if not fresh or fresh == url:
            return m.group(0)
        replaced[url] = fresh
        return f"{m.group(1)}{html.escape(fresh, quote=True)}{m.group(3)}"

    updated = _IMG_SRC_ATTR_RE.sub(_sub, content_html)
    if replaced:
        # Persist so a later view (and the list thumbnail) starts from the fresh
        # URL. It expires again in minutes, but by then the proxy has the bytes.
        try:
            with get_reader() as reader:
                db = reader._storage.get_db()
                for column in ("summary", "content"):
                    row = db.execute(
                        f"SELECT {column} FROM entries WHERE feed = ? AND id = ?",  # nosemgrep: literal
                        (feed_url, entry_id),
                    ).fetchone()
                    if not row or not row[0] or "wixmp" not in row[0]:
                        continue
                    body = row[0]
                    for old, new in replaced.items():
                        body = body.replace(html.escape(old, quote=True), html.escape(new, quote=True))
                        body = body.replace(old, new)
                    db.execute(
                        f"UPDATE entries SET {column} = ? WHERE feed = ? AND id = ?",  # nosemgrep: literal
                        (body, feed_url, entry_id),
                    )
                db.commit()
        except Exception:  # noqa: BLE001 — the served page is already correct
            LOGGER.exception("[deviantart] could not persist re-signed image for %s", entry_id)
        LOGGER.info("[deviantart] re-signed %d expired image URL(s) on open for %s", len(replaced), entry_id)
    return updated


_escalation_reprobe_running: set[str] = set()
_escalation_reprobe_lock = threading.Lock()


def _escalation_reprobe_for_user() -> None:
    """Re-probe flagged feeds for the bound user, then drop the flag caches so a cleared feed is fetched plainly at once."""
    # Mode "always" sends every fetch through the proxy by the user's choice; a direct probe would both test the wrong route and
    # expose the server's own IP to the sites the proxy is there for.
    if get_proxy_mode() == "always":
        return
    started = time.monotonic()

    def _invalidate_flag_caches(_feed_url: str) -> None:
        _invalidate_browser_ua_cache()
        _invalidate_proxy_feeds_cache()
        _invalidate_tailscale_feeds_cache()
        _invalidate_flaresolverr_feeds_cache()
        invalidate_problematic_feeds_cache()

    def _is_active(feed_url: str) -> bool:
        # A short-lived reader per check: the pass runs for hours and must not pin one.
        with get_reader() as reader:
            feed = reader.get_feed(feed_url, None)
        return feed is not None and bool(feed.updates_enabled)

    with get_meta_connection() as conn:
        result = escalation_reprobe.reprobe_flagged_feeds(
            conn,
            LECTIO_HONEST_USER_AGENT,
            is_active=_is_active,
            should_continue=lambda: get_proxy_mode() != "always",
            on_clear=_invalidate_flag_caches,
        )
    if result["probed"]:
        LOGGER.info(
            "[maintenance] escalation re-probe: %d probed in %.0fs, %d cleared, %d renewed",
            result["probed"],
            time.monotonic() - started,
            result["cleared"],
            result["renewed"],
        )


def _start_escalation_reprobe(uid: str) -> None:
    """Run _escalation_reprobe_for_user for *uid* on its own daemon thread, at most one at a time per user."""
    with _escalation_reprobe_lock:
        if uid in _escalation_reprobe_running:
            return
        _escalation_reprobe_running.add(uid)

    def _run() -> None:
        try:
            _run_in_user_context(uid, _escalation_reprobe_for_user)
        except Exception:
            LOGGER.exception("[maintenance] escalation re-probe failed for user %r", uid)
        finally:
            with _escalation_reprobe_lock:
                _escalation_reprobe_running.discard(uid)

    threading.Thread(target=_run, name="escalation-reprobe", daemon=True).start()


def _daily_maintenance_for_user() -> None:
    """Per-user nightly cleanup for the currently-bound tenancy user."""

    # 1. Prune rule_run_log older than 90 days. run_at is stored as a naive
    # ISO-8601 string (datetime.now().isoformat()), so compare against an ISO
    # cutoff — comparing the TEXT column to an int epoch never matched, and the
    # column was misnamed ran_at, so this prune silently never ran.
    try:
        cutoff = (datetime.now() - timedelta(days=90)).isoformat()
        with get_meta_connection() as conn:
            old_ids = [r[0] for r in conn.execute("SELECT id FROM rule_run_log WHERE run_at < ?", (cutoff,)).fetchall()]
            if old_ids:
                placeholders = ",".join("?" * len(old_ids))
                conn.execute(f"DELETE FROM rule_run_log_entries WHERE log_id IN ({placeholders})", old_ids)
                conn.execute(f"DELETE FROM rule_run_log WHERE id IN ({placeholders})", old_ids)
                LOGGER.info("[maintenance] pruned %d old rule run log entries", len(old_ids))
    except Exception:
        LOGGER.exception("[maintenance] rule log prune failed")

    # 1a. automation_dispatched claims only matter inside the dispatchers' 15-minute window.
    try:
        claim_cutoff = (datetime.now() - timedelta(days=7)).isoformat()
        with get_meta_connection() as conn:
            conn.execute("DELETE FROM automation_dispatched WHERE dispatched_at < ?", (claim_cutoff,))
    except Exception:
        LOGGER.exception("[maintenance] automation claim prune failed")

    # 1b. Bound feed_fetch_history: drop anything older than the age cap, plus a
    # fixed per-feed ceiling as a backstop against a pathologically-retried feed
    # (normal cadence — even hourly for 30 days — stays far under this; it exists
    # only so a genuine bug can't grow the table without limit).
    try:
        cutoff = time.time() - get_fetch_history_max_age_days() * 86400
        with get_meta_connection() as conn:
            cur = conn.execute(
                """
                DELETE FROM feed_fetch_history WHERE id IN (
                    SELECT id FROM (
                        SELECT id, ROW_NUMBER() OVER (
                            PARTITION BY feed_url ORDER BY fetched_at DESC
                        ) AS rn FROM feed_fetch_history
                    ) WHERE rn > ?
                )
                """,
                (FETCH_HISTORY_ROW_CEILING,),
            )
            pruned = cur.rowcount
            cur = conn.execute("DELETE FROM feed_fetch_history WHERE fetched_at < ?", (cutoff,))
            pruned += cur.rowcount
            if pruned:
                LOGGER.info("[maintenance] pruned %d feed fetch-history rows", pruned)
    except Exception:
        LOGGER.exception("[maintenance] fetch-history prune failed")

    # 1c. Per-folder retention: delete read posts N days after they were read
    # (folders.retention_days; starred/tagged posts and Saved Articles are
    # always protected — see _prune_entries).
    try:
        with get_meta_connection() as conn:
            folder_rows = conn.execute("SELECT id, name, retention_days FROM folders WHERE retention_days > 0").fetchall()
            retention_folder_feeds = {int(r["id"]): sorted(get_folder_feed_urls(conn, int(r["id"]))) for r in folder_rows}
        for row in folder_rows:
            feeds = retention_folder_feeds.get(int(row["id"])) or []
            if not feeds:
                continue
            days = int(row["retention_days"])
            deleted = _prune_entries(list(feeds), read_cutoff=datetime.now() - timedelta(days=days))
            # Always log (even 0) so a run is auditable — "nothing happened" and
            # "never ran" look identical otherwise.
            LOGGER.info("[maintenance] retention: deleted %d read post(s) (>%dd after read) from folder %s", deleted, days, row["name"])
    except Exception:
        LOGGER.exception("[maintenance] retention prune failed")

    # 1d. Tomb sweeping: drop deleted-entry tombstones older than the instance
    # setting (default 90 days, 0 = keep forever). By then the entries have
    # left every publisher's feed window, so resurrection is no longer a risk.
    try:
        sweep_days = get_tombstone_sweep_days()
        if sweep_days > 0:
            cutoff = (datetime.now() - timedelta(days=sweep_days)).isoformat()
            with get_meta_connection() as conn:
                # Only sweep tombstones whose entry has left the publisher's
                # last-known window — and only for feeds whose window we have
                # actually recorded (unknown window = keep, conservatively).
                # Age alone is NOT safe: a quiet YouTube channel 304s for
                # months while still carrying the same 15 videos, and its next
                # upload reparses the whole window.
                swept = conn.execute(
                    """
                    DELETE FROM deleted_entries
                    WHERE created_at IS NOT NULL AND created_at < ?
                      AND EXISTS (SELECT 1 FROM feed_seen_window w
                                  WHERE w.feed_url = deleted_entries.feed_url)
                      AND NOT EXISTS (SELECT 1 FROM feed_seen_window w2
                                      WHERE w2.feed_url = deleted_entries.feed_url
                                        AND w2.entry_id = deleted_entries.entry_id)
                    """,
                    (cutoff,),
                ).rowcount
            LOGGER.info("[maintenance] tomb sweep: removed %d tombstone(s) older than %dd and out of window", swept, sweep_days)
    except Exception:
        LOGGER.exception("[maintenance] tombstone sweep failed")

    # 1e. Hard-delete Saved Article husks apply_star_state deliberately left
    # behind (unstarred+untagged, but only once their undo window — if any —
    # has passed). See _sweep_husked_saved_articles.
    try:
        husked = _sweep_husked_saved_articles()
        if husked:
            LOGGER.info("[maintenance] husk sweep: hard-deleted %d Saved Article husk(s)", husked)
    except Exception:
        LOGGER.exception("[maintenance] husk sweep failed")

    # 2. Purge orphaned meta DB rows (feeds that no longer exist in the reader).
    try:
        with get_reader() as reader:
            live_urls = {f.url for f in reader.get_feeds()}
        with get_meta_connection() as conn:
            for table, col in [
                ("feed_strategy_cache", "feed_url"),
                ("feed_display_prefs", "feed_url"),
                ("feed_failure_state", "feed_url"),
            ]:
                conn.execute(
                    f"DELETE FROM {table} WHERE {col} NOT IN (SELECT feed_url FROM folder_feeds)",
                )
            # domain_failure_state: remove domains with no remaining feeds
            live_domains = {urlparse(u).netloc.lower() for u in live_urls}
            rows = conn.execute("SELECT domain FROM domain_failure_state").fetchall()
            stale_domains = [r["domain"] for r in rows if r["domain"] not in live_domains]
            if stale_domains:
                conn.executemany(
                    "DELETE FROM domain_failure_state WHERE domain = ?",
                    [(d,) for d in stale_domains],
                )
        LOGGER.info("[maintenance] orphaned row cleanup done")
    except Exception:
        LOGGER.exception("[maintenance] orphan cleanup failed")

    # 2b. Sweep unrecoverable failed-archive rows so the Stats "failed" count
    # self-heals. A failed capture is only orphaned — never retryable — once its
    # entry is gone from reader AND no star keeps it; a failed capture of a live,
    # still-starred entry is left for a later retry.
    try:
        with get_reader() as reader, get_meta_connection() as conn:

            def _keep(feed_url: str, entry_id: str) -> bool:
                if conn.execute(
                    "SELECT 1 FROM saved_entries WHERE feed_url = ? AND entry_id = ?",
                    (feed_url, entry_id),
                ).fetchone():
                    return True
                return reader.get_entry((feed_url, entry_id), None) is not None

            swept = starred_archive_service.sweep_failed_orphans(_keep)
        if swept:
            LOGGER.info("[maintenance] archive orphan sweep: removed %d unrecoverable failed capture(s)", swept)
    except Exception:
        LOGGER.exception("[maintenance] archive orphan sweep failed")

    # 2c. Flag captures that failed transiently (timeout, connect, 5xx) for retry: 20 rows a night, 3 per host, each row at most
    # 3 times, 7+ days apart.
    # Rows whose entry has left the reader (unsubscribed feeds) have nothing to re-capture and are skipped.
    try:
        with get_reader() as reader:
            requeued = starred_archive_service.requeue_transient_failures(
                lambda feed_url, entry_id: reader.get_entry((feed_url, entry_id), None) is not None
            )
        if requeued:
            LOGGER.info("[maintenance] archive retry: flagged %d transient capture failure(s)", requeued)
    except Exception:
        LOGGER.exception("[maintenance] archive retry failed")

    # 3. VACUUM this user's own SQLite DBs (the shared thumb cache is global and
    # vacuumed once in _run_global_maintenance).
    for label, path in [
        ("meta", tenancy.meta_db_path()),
        ("starred-archive", tenancy.starred_archive_db_path()),
    ]:
        try:
            if not Path(path).exists():
                continue
            conn = sqlite3.connect(str(path))
            conn.execute("VACUUM")
            # Fold the vacuum's WAL (≈ DB-sized) back and truncate it — with the
            # app's pooled connections holding the DB open it lingers otherwise.
            truncate_wal(conn, f"VACUUM {label}")
            conn.close()
            LOGGER.info("[maintenance] VACUUM %s done", label)
        except Exception:
            LOGGER.exception("[maintenance] VACUUM %s failed", label)

    # 4. Flush pending email batch queues.
    try:
        _flush_all_email_batches()
        LOGGER.info("[maintenance] email batch flush done")
    except Exception:
        LOGGER.exception("[maintenance] email batch flush failed")

    # 5. YouTube subscription sync — per-user (this user's own key + channel).
    if get_yt_api_key() and get_yt_channel_id():
        try:
            result = _run_youtube_sync()
            if result.get("error"):
                LOGGER.error("[maintenance] YouTube sync error: %s", result["error"])
            else:
                LOGGER.info("[maintenance] YouTube sync: +%d -%d total=%d", result["added"], result["removed"], result["total"])
        except Exception:
            LOGGER.exception("[maintenance] YouTube sync failed")

    # 6. DeviantArt watch-list → gallery feeds sync — per-user, only when the
    # account is connected. Mirrors the YouTube sync above so the watch list no
    # longer needs the manual Settings button. sync_deviantart_watchlist is
    # add-only and self-throttles against DeviantArt's rate limit.
    if get_deviantart_user_token():
        try:
            result = sync_deviantart_watchlist()
            rate_suffix = " (rate limited)" if result.get("rate_limited") else ""
            if result.get("error"):
                LOGGER.error("[maintenance] DeviantArt watch-list sync error%s: %s", rate_suffix, result["error"])
            else:
                LOGGER.info(
                    "[maintenance] DeviantArt watch-list sync: +%d watched=%d%s",
                    result.get("added", 0),
                    result.get("total", 0),
                    rate_suffix,
                )
        except Exception:
            LOGGER.exception("[maintenance] DeviantArt watch-list sync failed")
        # 6b. Re-check parked deactivated artists for reactivation.
        try:
            reactivated = _deviantart_recheck_deactivated()
            if reactivated:
                LOGGER.info("[maintenance] DeviantArt: %d deactivated artist(s) reactivated and subscribed", reactivated)
        except Exception:
            LOGGER.exception("[maintenance] DeviantArt deactivated re-check failed")

    # 7. Record this user's last-ran timestamp.
    with get_meta_connection() as conn:
        set_setting(conn, "maintenance_last_ran_at", time.strftime("%Y-%m-%d %H:%M %Z"))

    # 8. Step feeds back down the fetch-escalation ladder (see services/escalation_reprobe.py). Last, so it does not contend with the
    # VACUUMs above for the meta DB, and off this thread: a full pass can run for hours, and this thread also flushes email batches
    # and rechecks premieres.
    _start_escalation_reprobe(tenancy.current_user_id())


def _evict_img_cache() -> None:
    """Drop /api/img cache entries not served within the configured TTL. The TTL
    is on last_accessed (not created_at), so actively-browsed images stay cached
    and only stale/long-unread ones age out. 0 days = keep forever (no eviction)."""
    days = get_img_cache_days()
    if days <= 0:
        return
    cutoff = time.time() - days * 86400
    try:
        with get_img_cache_connection() as conn:
            cur = conn.execute(
                "DELETE FROM img_cache WHERE last_accessed < ? AND cache_key NOT LIKE ? AND cache_key NOT LIKE ?",
                (cutoff, _FEED_THUMB_CACHE_PREFIX + "%", _ENTRY_THUMB_CACHE_PREFIX + "%"),
            )
            LOGGER.info("[maintenance] img cache: evicted %d entries older than %d days", cur.rowcount, days)
    except Exception:
        LOGGER.exception("[maintenance] img cache eviction failed")


def _run_global_maintenance() -> None:
    """Nightly cleanup that is not per-user: evict stale /api/img cache entries,
    then VACUUM the shared (global) caches."""
    _evict_img_cache()
    for label, path in (
        ("thumb", THUMB_DB_PATH),
        ("img-cache", IMG_CACHE_DB_PATH),
        ("yt-durations", YT_DURATION_DB_PATH),
    ):
        try:
            conn = sqlite3.connect(str(path))
            conn.execute("VACUUM")
            # Fold the vacuum's WAL (≈ DB-sized) back and truncate it — with the
            # app's pooled connections holding the DB open it lingers otherwise.
            truncate_wal(conn, f"VACUUM {label}")
            conn.close()
            LOGGER.info("[maintenance] VACUUM %s done", label)
        except Exception:
            LOGGER.exception("[maintenance] VACUUM %s failed", label)


# Instance-wide "maintenance last ran on this date" marker. A file in the data
# dir (not a per-user DB setting) so the once-a-day decision survives restarts
# and isn't tied to any tenant — used for the catch-up check below.
def _maint_marker_path() -> Path:
    return DATA_DIR / "maintenance_last_run_date"


def _maint_last_run_date() -> str:
    try:
        return _maint_marker_path().read_text(encoding="utf-8").strip()
    except OSError:
        return ""


def _set_maint_last_run_date(date_str: str) -> None:
    try:
        _maint_marker_path().write_text(date_str, encoding="utf-8")
    except OSError:
        LOGGER.warning("[maintenance] could not persist last-run date", exc_info=True)


def _maintenance_due(now_hour: int, maint_hour: int | None, last_run_date: str, today: str) -> bool:
    """True when daily maintenance should run now: enabled, at or past the
    configured hour, and not already run today (catch-up if the exact hour was
    missed)."""
    if maint_hour is None:
        return False
    return now_hour >= maint_hour and last_run_date != today


def _daily_maintenance_loop(stop_event: threading.Event) -> None:
    """Thread that fires _run_daily_maintenance() once per day and flushes email
    batch queues at their configured batch_time each minute.

    Catch-up semantics: maintenance runs once per calendar day as soon as the
    app is alive AT OR AFTER the configured hour and hasn't run today — not only
    during the exact hour. A restart or deploy near the maintenance hour (or a
    machine that was asleep) used to skip the whole day; now it runs at the next
    opportunity. The last-run date is persisted (data-dir marker) so restarts
    don't re-run the same day."""
    last_batch_check_hhmm: str | None = None
    last_premiere_recheck_at = 0.0
    while not stop_event.wait(30):
        # Flush email batches at their configured batch_time (once per clock
        # minute), per user — email_batch_queue lives in each user's meta DB.
        now_hhmm = time.strftime("%H:%M")
        if now_hhmm != last_batch_check_hhmm:
            last_batch_check_hhmm = now_hhmm
            for _uid in _background_user_ids():
                with tenancy.user_context(_uid):
                    try:
                        _check_and_flush_batch_times()
                    except Exception:
                        LOGGER.exception("[maintenance] email batch flush failed for user %r", _uid)

        # Recheck still-"upcoming" YouTube videos hourly, independent of any
        # feed's own refresh cadence — a channel that refreshes rarely could
        # otherwise leave a premiere's status stale for hours after it airs.
        # Fixed interval rather than a clock-hour boundary: "roughly hourly"
        # is the point, not exactness, and this sidesteps DST/timezone noise.
        now_mono = time.monotonic()
        if now_mono - last_premiere_recheck_at >= 3600:
            last_premiere_recheck_at = now_mono
            for _uid in _background_user_ids():
                with tenancy.user_context(_uid):
                    try:
                        _checked = youtube_duration_service.refresh_upcoming_videos()
                        if _checked:
                            LOGGER.info("[maintenance] rechecked %d upcoming YouTube video(s)", _checked)
                    except Exception:
                        LOGGER.exception("[maintenance] YouTube premiere recheck failed for user %r", _uid)

        # Daily maintenance: once per day, at or after the configured hour.
        maint_hour = get_maintenance_hour()
        if maint_hour is None:
            continue
        now_lt = time.localtime()
        today = time.strftime("%Y-%m-%d")
        if _maintenance_due(now_lt.tm_hour, maint_hour, _maint_last_run_date(), today):
            _set_maint_last_run_date(today)  # claim the slot before running
            try:
                _run_daily_maintenance()
            except Exception:
                LOGGER.exception("[maintenance] unhandled error in daily maintenance")
            stop_event.wait(61)  # skip rest of the minute


def check_and_mark_manual_refresh() -> int:
    return check_manual_refresh_cooldown(MANUAL_REFRESH_COOLDOWN_SECONDS)


# Startup/shutdown handled by the lifespan() context manager above.


def _resolve_archived_readability_html(feed_url: str | None, entry_id: str | None) -> str | None:
    """Return the offline archived readability HTML for a starred entry with its
    local image assets rewritten to /starred-asset/ URLs, or None when no
    complete archive exists. Shared by the reader-view route and the e-ink
    ``/read`` view so both prefer the archived copy over a live re-fetch."""
    if not (feed_url and entry_id):
        return None
    archived_html = starred_archive_service.get_archived_readability_html(feed_url, entry_id)
    if not archived_html:
        return None
    asset_map = starred_archive_service.get_entry_asset_map(feed_url, entry_id)
    if asset_map:
        archived_html = starred_archive_service.rewrite_html_assets(archived_html, asset_map, STARRED_ASSET_URL_PREFIX)
    return archived_html


# ---------------------------------------------------------------------------
# E-ink reader view (Supernote Manta) — standalone, paginated, distraction-free
#
# A self-contained reading surface for the saved/starred backlog: its own HTML +
# static/reader.{css,js} (no app shell), high contrast, no scrolling (tap
# left/right or arrow keys turn CSS-column "pages"). It follows the Saved view's
# current filter and prefers the offline archived readability copy over a
# truncated feed summary. See ARCHITECTURE "E-ink reader view".
# ---------------------------------------------------------------------------


def _mark_entry_read_background(feed_url: str, entry_id: str, title: str, link: str, feed_title: str) -> None:
    """Mark an entry read off the request path. The daemon thread does not
    inherit the request's tenancy contextvars, so capture the current user now
    and re-bind it via _run_in_user_context — otherwise the write has no user
    and raises TenancyUnboundError."""
    _uid = tenancy.current_user_id()

    def _bg_mark_read() -> None:
        try:
            with get_reader() as reader:
                reader.mark_entry_as_read((feed_url, entry_id))
        except Exception:
            LOGGER.warning("background mark_entry_as_read failed for %s/%s", feed_url, entry_id, exc_info=True)
        try:
            upsert_entry_read_state(feed_url, entry_id)
        except Exception:
            LOGGER.warning("background upsert_entry_read_state failed for %s/%s", feed_url, entry_id, exc_info=True)
        try:
            append_read_history(feed_url, entry_id, title, link, feed_title)
        except Exception:
            LOGGER.warning("background append_read_history failed for %s/%s", feed_url, entry_id, exc_info=True)
        invalidate_unread_counts_cache()

    threading.Thread(target=_run_in_user_context, args=(_uid, _bg_mark_read), daemon=True).start()


def _prepend_reader_lead_image(feed_url: str | None, entry_id: str | None, body: str) -> str:
    """Prepend the entry's cached lead/hero image to the reader body when it isn't
    already there. Readability extraction usually strips the opening image (Lectio
    tracks it separately as the lead image), so without this the first image is
    missing in Read Mode — the normal reader view re-adds it client-side."""
    body = proxy_all_body_images(body)
    if not (feed_url and entry_id):
        return body
    try:
        lead = lead_image_service.get_cached_lead_image_url(feed_url, entry_id)
    except Exception:
        lead = None
    # Cached leads outlive the rules that chose them: a row stored before ad and
    # social-icon rejection existed would still be prepended here, so the reader
    # showed decibelmagazine's "…-hero-superbanner.gif" above the article even
    # after the body strip removed the mid-article copy. Checked at render, which
    # also means a stale row is harmless rather than needing a sweep.
    if lead and (lead_image_service.is_ad_url(lead) or lead_image_service.is_social_icon_url(lead)):
        return body
    if not lead or lead in body or html.escape(lead, quote=True) in body:
        return body
    lead_html = f'<p class="reader-lead"><img src="{html.escape(lead, quote=True)}" alt="" loading="lazy"></p>'
    # Route through the same hotlink/no-referrer handling as article images.
    lead_html = add_no_referrer_to_images(proxy_hotlink_images(lead_html))
    return lead_html + body


# Below this many characters of *text*, an archived readability copy is treated as
# a failed extraction rather than a short article. Chosen to clear a sidebar widget
# ("Contact: …" at 168 bytes) while sitting well under any real post; a genuinely
# short article still renders, because the archive is kept as the fallback when
# nothing richer exists.
_MIN_ARCHIVED_ARTICLE_TEXT = 400


_READER_TEXT_TAG_RE = re.compile(r"<[^>]{1,10000}>")


def _reader_text_length(html_text: str | None) -> int:
    """Visible-text length of a fragment — tags carry no reading value, so a
    markup-heavy widget must not out-measure real prose.

    The tag-strip is bounded (``{1,10000}`` rather than ``+``): an unbounded
    quantifier ahead of a required ``>`` re-scans to the end of the string at
    every unmatched ``<`` in feed-supplied HTML, which is O(n^2) on adversarial
    input (ReDoS). No real tag is anywhere near 10000 chars.
    """
    if not html_text:
        return 0
    return len(re.sub(r"\s+", " ", _READER_TEXT_TAG_RE.sub(" ", html_text)).strip())


def _reader_img_count(html_text: str | None) -> int:
    return len(re.findall(r"<img\b", html_text or "", re.IGNORECASE))


def _archived_copy_is_plausible(html_text: str) -> bool:
    """Does an archived readability copy look like a real article body?

    Text length alone is not enough. A picture post is legitimately almost text-
    free — illogicalcontraption's 2011 entry is 16 images and 57 characters — so
    images count as substance in their own right, or every comic and photo post
    would be judged a failed extraction.
    """
    return _reader_text_length(html_text) >= _MIN_ARCHIVED_ARTICLE_TEXT or _reader_img_count(html_text) >= 2


def _reader_copy_is_richer(candidate: str, current: str) -> bool:
    """Is *candidate* more of an article than *current*? Images first, then text.

    Images lead because the failure being corrected is a text widget beating a
    picture post; on that comparison a text-length test came down to 57 characters
    against 47, which is not a margin to trust.
    """
    cand = (_reader_img_count(candidate), _reader_text_length(candidate))
    cur = (_reader_img_count(current), _reader_text_length(current))
    return cand > cur


_BEACON_IMG_RE = re.compile(
    r"""<img\b[^>]*?\bsrc\s*=\s*["']([^"']+)["'][^>]*>""",
    re.IGNORECASE | re.DOTALL,
)


def _drop_feed_beacon_images(content: str) -> str:
    """Remove FeedBurner-style tracking beacons and share icons from a body.

    They are invisible counters, and the services behind them are dead —
    feeds.feedburner.com/~ff/... answers 404 with an HTML error page. Left in,
    every one becomes a proxied fetch that can only ever fail: /api/img rejects
    the non-image response with a 422, so a single article logged four of them on
    every view. Nothing is lost by dropping them; they were never visible.
    """
    if "<img" not in content.lower():
        return content

    def _drop(m: re.Match) -> str:
        src = html.unescape(m.group(1))
        return "" if link_canonical.is_redirector_link(src) else m.group(0)

    return _BEACON_IMG_RE.sub(_drop, content)


_READER_IMG_SRC_RE = re.compile(r'(<img\b[^>]*?\bsrc\s*=\s*)(["\'])(.*?)\2', re.IGNORECASE | re.DOTALL)


def proxy_all_body_images(content: str) -> str:
    """Point every remote <img> in an article body at /api/img.

    Read Mode (e-ink reader) always runs this: offline reading needs the page
    to request the SAME URLs the precache manifest stores, and the manifest can
    only list same-origin ones — so an article whose images are absolute
    (blogspot, most feeds) precached its HTML and none of its pictures, and
    read fine with every image broken.

    The main article pane runs this too, opt-in via proxy_body_images_enabled()
    (Settings → Account → Appearance): it stops content blockers that filter a
    third-party image CDN from leaving the article looking empty, and brings
    body images into the same cache heroes already use.

    Proxying also removes a quieter dependency either way: those srcs are
    frequently ``http://`` on an ``https://`` page, and they load today only
    because Chrome silently upgrades them. That upgrade needs the network,
    which is precisely what offline does not have.

    Same guard, cache and user-agent as any other /api/img load. srcset is
    dropped alongside, or the browser picks a direct URL over the proxied src
    (defeating the manifest offline, and high-DPI screens fetch a single
    resolution instead when this runs in the main pane).
    """
    if "<img" not in content.lower():
        return content

    def _rewrite(m: re.Match) -> str:
        prefix, quote_ch, src = m.group(1), m.group(2), html.unescape(m.group(3).strip())
        if not src or src.startswith(("data:", "/")):
            return m.group(0)
        if not src.lower().startswith(("http://", "https://")):
            return m.group(0)
        return f"{prefix}{quote_ch}/api/img?u={quote(src, safe='')}{quote_ch}"

    content = _drop_feed_beacon_images(content)
    out = _READER_IMG_SRC_RE.sub(_rewrite, content)
    # Bounded attribute-value quantifiers ({0,10000}, not *): an unbounded
    # [^"]*/[^']* ahead of a required closing quote re-scans to end-of-string
    # for every unclosed attribute in feed-supplied HTML — O(n^2) on
    # adversarial input (ReDoS). No real attribute value is anywhere near
    # 10000 chars.
    return re.sub(
        r"\s+(?:srcset|data-srcset|data-src|data-lazy-src)\s*=\s*"
        r'(?:"[^"]{0,10000}"|\'[^\']{0,10000}\')',
        "",
        out,
        flags=re.IGNORECASE,
    )


def resolve_reader_article_html(feed_url: str | None, entry_id: str | None, link: str) -> str:
    """Return sanitized article HTML for the e-ink reader, preferring the offline
    archived readability copy, then a live readability extraction of *link*, then
    the stored feed content. Every source is already sanitized (archive at
    capture, live via sanitize_readability_html, stored via reader_sanitize at
    ingest), so the caller embeds the result directly. The entry's lead image is
    prepended (readability strips it; Lectio tracks it separately)."""
    archived_html = _strip_ad_images(_resolve_archived_readability_html(feed_url, entry_id))
    if archived_html and _archived_copy_is_plausible(archived_html):
        return _prepend_reader_lead_image(feed_url, entry_id, _strip_bandcamp_track_signature(archived_html))
    # An implausibly short (or entirely empty) archived copy is a FAILED
    # extraction, not a short article: readability sometimes locks onto a
    # sidebar widget instead of the body, and the recapture mismatch guard
    # leaves readability_html empty outright rather than store a
    # parked/replaced page. illogicalcontraption's 2011 post archived as a
    # 168-byte "Contact:" block while the feed held 9,208 bytes of the actual
    # post — and because the archive won unconditionally, the article read as
    # empty online and offline alike. Prefer the stored feed content when it
    # is genuinely richer, or whenever the archive came back with nothing at
    # all. Scoped to entries with a complete archive (kept, capture already
    # tried) rather than just "archived_html is falsy" — an ordinary entry
    # with no archive at all must still reach the live fetch below, which is
    # what recovers a full article from a thin RSS stub in the common case.
    if feed_url and entry_id and starred_archive_service.has_complete_archive(feed_url, entry_id):
        detail = get_entry_detail(feed_url, entry_id)
        stored = str((detail or {}).get("content_html") or "")
        if stored and (not archived_html or _reader_copy_is_richer(stored, archived_html)):
            return _prepend_reader_lead_image(feed_url, entry_id, stored)
    if link:
        try:
            _title, article_html = fetch_readability_article(link)
            # A 200 does not mean the article is still there -- a URL folded
            # into a generic category/landing page extracts as a list of
            # unrelated links, not an article (guitarworld.com's retired
            # /lessons/<slug> URLs all now 301 to one "Lessons Coverage" hub).
            # Fall through to stored content below rather than show it.
            if article_html and not saved_articles_service.looks_like_a_link_index(article_html):
                return _prepend_reader_lead_image(feed_url, entry_id, article_html)
        except Exception:
            LOGGER.info("reader-view live extraction failed for %s", link, exc_info=True)
    if feed_url and entry_id:
        detail = get_entry_detail(feed_url, entry_id)
        if detail and detail.get("content_html"):
            return _prepend_reader_lead_image(feed_url, entry_id, str(detail["content_html"]))
    if archived_html:
        # Thin, but better than nothing — a genuinely short article ends up here.
        return _prepend_reader_lead_image(feed_url, entry_id, _strip_bandcamp_track_signature(archived_html))
    esc = html.escape(link or "", quote=True)
    tail = f" <a href='{esc}' target='_blank' rel='noopener noreferrer'>Open original</a>." if link else ""
    return f"<p>Could not load this article.{tail}</p>"


def resolve_reader_backlog(
    *,
    folder_id: int | None,
    list_feed_url: str | None,
    read_filter: str,
    star_only: bool,
    tag: str | None,
    sort_by: str,
    sort_dir: str,
    search_query: str | None,
    archived: bool | None = None,
    limit: int = 250,
    kept_scope: str = "kept",
) -> list[dict]:
    """Ordered entry backlog for the reader, mirroring the main list view's
    feed-set selection so the reader can "follow the Saved view filter": the root
    folder widens to every reader feed, a specific folder/feed narrows to it, and
    archive-only saved orphans are merged into the star_only root view (matching
    the Saved Articles list).

    ``archived`` is the Read Mode axis (independent of read/unread): ``False``
    hides Archived items (the inbox), ``True`` keeps only Archived items, ``None``
    applies no archive filter.

    ``kept_scope`` picks which keep signals count as "saved" — ``"kept"``
    (starred OR tagged, the main app's Saved view) or ``"starred"`` (Read Mode's
    Inbox, which is a to-do pile rather than the whole library)."""
    with get_meta_connection() as conn:
        snapshot = get_meta_structure_snapshot(conn)
        disabled_feed_urls = get_disabled_feed_urls(conn)
    all_feed_urls = cast(set[str], snapshot["all_feed_urls"])
    root_id = cast(int, snapshot["root_id"])
    folder_feed_urls_by_id = dict(cast("dict[int, set[str]]", snapshot["folder_feed_urls_by_id"]))
    all_reader_feed_urls = get_all_reader_feed_urls()
    # lectio:saved is a real reader feed (backs the Saved/Kept view) but must
    # never be reachable as an ordinary Feeds-mode subscription — same
    # exclusion get_folder_feed_urls and _home_inner apply. Found 2026-08-28:
    # this resolver independently rebuilds its own root/Uncategorized feed
    # sets rather than reading get_folder_feed_urls's, so it needs its own copy
    # of the exclusion, or Feeds-mode Read Mode over Uncategorized surfaced the
    # whole saved-articles backlog.
    folder_feed_urls_by_id[root_id] = set(all_reader_feed_urls) - {saved_articles_service.SAVED_FEED_URL}
    folder_feed_urls_by_id[UNCATEGORIZED_FOLDER_ID] = all_reader_feed_urls - all_feed_urls - {saved_articles_service.SAVED_FEED_URL}

    selected_folder_id = folder_id or root_id
    if list_feed_url:
        entry_feed_urls = {list_feed_url}
    elif star_only:
        # Stars are deliberate curation: keep a disabled feed's saved items
        # visible (the saved badges/counts include them).
        entry_feed_urls = set(folder_feed_urls_by_id.get(selected_folder_id, set()))
        if selected_folder_id in (root_id, UNCATEGORIZED_FOLDER_ID):
            # Saved mode's own reachability into lectio:saved — same
            # star_only-gated re-inclusion _home_inner uses. It's an orphan
            # feed like any other, so it belongs at Saved's root and in its
            # Uncategorized grouping, just never as a browsable subscription.
            entry_feed_urls = entry_feed_urls | {saved_articles_service.SAVED_FEED_URL}
    else:
        entry_feed_urls = set(folder_feed_urls_by_id.get(selected_folder_id, set())) - disabled_feed_urls

    posts = list_entries_for_feeds(
        entry_feed_urls,
        limit=limit,
        sort_by=sort_by,
        sort_dir=sort_dir,
        read_filter=read_filter,
        star_only=star_only,
        selected_tag=tag,
        search_query=search_query,
        kept_scope=kept_scope,
        archived=archived,
    )

    # Parity with the Saved list: surface archive-only orphans (saves whose feed
    # was unsubscribed) in the whole-backlog star view — root, no feed/tag. A
    # search query narrows them (metadata match) rather than excluding them —
    # orphans have no reader row for the SQL search path to reach otherwise.
    _merged_orphans = False
    if star_only and not list_feed_url and not tag and selected_folder_id == root_id:
        try:
            posts = merge_orphan_saved_entries(
                posts,
                live_feed_urls=all_feed_urls,
                sort_by=sort_by,
                sort_dir=sort_dir,
                limit=limit,
                archived=archived,
                search_terms=search_terms_from_query(search_query),
                kept_scope=kept_scope,
                capture_failed=normalize_read_filter(read_filter) == "capture_failed",
            )
            _merged_orphans = True
        except Exception as exc:  # noqa: BLE001
            LOGGER.warning("reader backlog orphan merge failed: %s", exc)

    if archived is not None and not _merged_orphans:
        # Both list_entries_for_feeds and merge_orphan_saved_entries apply this
        # before their own clips; this covers the path where the merge is skipped.
        archived_keys = get_archived_saved_keys()
        posts = [p for p in posts if ((str(p["feed_url"]), str(p["id"])) in archived_keys) == archived]
    return posts


def _read_scope_params(
    folder_id: int | None,
    tag: str | None,
    archived: bool,
    q: str | None,
    scope: str = "saved",
    list_feed_url: str | None = None,
    sort: str | None = None,
    resume_sort: str | None = None,
    kept_all: bool = False,
) -> list[tuple[str, str]]:
    """Read Mode node scope shared by the browse URL and the reader prev/next
    URLs (scope / folder / feed / tag / Archive / search / sort).

    *sort* rides along so the chosen order survives every hop — pick Oldest in
    the browse pane and Next/Prev walk oldest-first too, instead of silently
    reverting to the default at the first navigation.

    *resume_sort* is the order to restore on leaving the Inbox. The Inbox sorts
    by most-recently-starred, which is meaningless anywhere else, so it must not
    follow you out — but neither should leaving reset a folder you had set to
    Oldest. Same shape as the main app's ``resume_read_filter``, which restores
    your previous filter when you close History."""
    params: list[tuple[str, str]] = []
    if scope and scope != "saved":
        params.append(("scope", scope))
    if folder_id is not None:
        params.append(("folder_id", str(folder_id)))
    if list_feed_url:
        params.append(("list_feed_url", list_feed_url))
    if tag:
        params.append(("tag", tag))
    if archived:
        params.append(("archived", "1"))
    if q:
        params.append(("q", q))
    if kept_all:
        params.append(("kept", "all"))
    if sort and sort != _READ_SORT_DEFAULT:
        params.append(("sort", sort))
    if resume_sort and resume_sort != _READ_SORT_DEFAULT:
        params.append(("resume_sort", resume_sort))
    return params


def _read_browse_href(
    folder_id: int | None,
    tag: str | None,
    archived: bool,
    q: str | None,
    scope: str = "saved",
    list_feed_url: str | None = None,
    sort: str | None = None,
    resume_sort: str | None = None,
    kept_all: bool = False,
) -> str:
    """The 2-pane browse URL for a Read Mode node (no entry selected)."""
    params = _read_scope_params(folder_id, tag, archived, q, scope, list_feed_url, sort, resume_sort, kept_all)
    return "/read" + ("?" + urlencode(params) if params else "")


def _reader_href(
    feed_url: str,
    entry_id: str,
    *,
    folder_id: int | None,
    tag: str | None,
    archived: bool,
    q: str | None,
    scope: str = "saved",
    list_feed_url: str | None = None,
    sort: str | None = None,
    kept_all: bool = False,
) -> str:
    """Build a /read URL that opens *entry* in the reader while carrying the
    current Read Mode node scope, so prev/next stay within the same list."""
    params = [("feed_url", feed_url), ("entry_id", entry_id)]
    params += _read_scope_params(folder_id, tag, archived, q, scope, list_feed_url, sort, None, kept_all)
    return "/read?" + urlencode(params)


def _reader_empty_response() -> HTMLResponse:
    doc = (
        "<!DOCTYPE html><html lang='en'><head><meta charset='utf-8'>"
        "<title>Reader</title>"
        "<meta name='viewport' content='width=device-width, initial-scale=1'>"
        f"<link rel='stylesheet' href='/static/reader.css?v={STATIC_ASSET_VERSION}'>"
        "</head><body><div class='reader-empty'>"
        "<h1>Nothing to read</h1><p>There are no articles in this view.</p>"
        "<p><a href='/'>Back to Lectio</a></p>"
        "</div></body></html>"
    )
    return HTMLResponse(doc, headers={"Cache-Control": "no-store"})


def build_reader_page(
    *,
    title: str,
    article_html: str,
    source_link: str,
    prev_href: str,
    next_href: str,
    back_href: str,
    feed_url: str,
    entry_id: str,
    is_archived: bool,
    csrf_token: str,
    show_saved_actions: bool = True,
    date_display: str = "",
    manual_tags: tuple[str, ...] = (),
    all_tag_names: tuple[str, ...] = (),
    katex_dollar_math: bool = False,
) -> HTMLResponse:
    # Same allowlist as the list rows: the feed's <em> renders, and a literal
    # <T> or <chrono> in a C++ title stays visible text.
    esc_title = html_sanitize.sanitize_inline_title(title or "(untitled)")
    # <title> is RCDATA — markup does not render there, it shows as tag text. It
    # gets the tag-stripped, escaped form instead (what the other reader builder
    # already does).
    head_title = html.escape(html_sanitize.title_plain_text(title or "(untitled)"))
    esc_src = html.escape(source_link or "", quote=True)
    esc_back = html.escape(back_href or "/read", quote=True)
    esc_feed = html.escape(feed_url or "", quote=True)
    esc_eid = html.escape(entry_id or "", quote=True)
    esc_csrf = html.escape(csrf_token or "", quote=True)
    reader_dateline = f"<p class='reader-dateline'>{html.escape(date_display)}</p>" if date_display else ""
    open_original = (
        f"<a class='reader-ctl' href='{esc_src}' target='_blank' rel='noopener noreferrer' title='Open original'>&#8599;</a>"
        if source_link
        else ""
    )
    # Archive shows a filled box when already archived (tap = un-archive). Saved
    # actions (Archive/Delete) only apply to the saved scope; the feeds scope is
    # ordinary unread reading. Both scopes now mark read on reaching the last
    # page rather than on open — the peek problem is the same in either, and
    # splitting the rule by scope would make "did that count as read?"
    # unpredictable from inside the reader, where the scope isn't visible.
    # ▤ (U+25A4) is the glyph the Read Mode tree already uses for Archive.
    # The old ▢/▣ pair read as an empty checkbox on e-ink — the shape said
    # nothing about archiving, and the filled/empty difference was invisible
    # at that size. State is now shown by inverting the button (see
    # reader.css [aria-pressed]), which e-ink renders crisply.
    archive_glyph = "&#9636;"
    archive_title = "Un-archive" if is_archived else "Archive"
    # Always shown now. Archive used to be hidden for a tag-kept item, because
    # the done-flag lived on the star row and an item kept only by a tag had
    # nothing to set. The done-axis has its own table (archived_entries), so
    # anything in the inbox can be marked done — which matters: tagged entries
    # outnumber starred ones 16,479 to 10,002, and they were the ones with no
    # way to be cleared from the device.
    archive_btn = (
        f"<button class='reader-ctl' id='reader-archive-btn' type='button'"
        f" aria-pressed='{'true' if is_archived else 'false'}' title='{archive_title}'>{archive_glyph}</button>"
    )
    # Delete removes the item from Kept entirely — star *and* tags. The tag list
    # rides along so the confirm can name what is about to be lost.
    esc_tags = html.escape(",".join(manual_tags), quote=True)
    delete_btn = (
        "<button class='reader-ctl' id='reader-delete-btn' type='button'"
        f" data-tags='{esc_tags}' title='Delete (remove from Saved)'>&#128465;</button>"
    )
    # "#" rather than a 🏷 emoji: emoji render inconsistently (and sometimes in
    # colour) on e-ink browsers, while "#" is the same mark Lectio uses for tags
    # everywhere else.
    tag_btn = (
        "<button class='reader-ctl' id='reader-tag-btn' type='button'"
        f" aria-expanded='false' title='Tags'>#{len(manual_tags) or ''}</button>"
    )
    # Download this article as a self-contained file. A spare route, not the main
    # one — the service worker is what offline reading runs on — but a plain
    # <a download> with no JS is the one thing that cannot break on an old
    # WebView, so it stays. ⤓ (U+2913)
    offline_btn = (
        "<a class='reader-ctl' id='reader-offline-btn'"
        f" href='/read/offline?feed_url={quote_plus(feed_url)}&entry_id={quote_plus(entry_id)}'"
        " download title='Save local copy'>&#10515;</a>"
    )
    saved_actions = (tag_btn + archive_btn + delete_btn + offline_btn) if show_saved_actions else ""
    # Prev/next/back navigation targets are passed as an inline JS object rather
    # than data-* attributes so reader.js never reads them from the DOM (avoids a
    # CodeQL js/xss-through-dom false positive on location.assign). Values are
    # server-generated /read paths; JSON-encode + escape "<" against script
    # breakout.
    nav_json = json.dumps({"prev": prev_href or "", "next": next_href or "", "back": back_href or "/read"}).replace("<", "\\u003c")
    # The tag vocabulary rides in the same inline-JSON style, for the same reason:
    # nothing the panel acts on is read back out of the DOM. `all` is every tag
    # name in the library so the panel is a tap-list rather than a keyboard task;
    # `current` is what this entry carries.
    tags_json = json.dumps(
        {"all": list(all_tag_names), "current": list(manual_tags), "feed_url": feed_url, "entry_id": entry_id, "max": MAX_MANUAL_TAGS}
    ).replace("<", "\\u003c")
    # Rendered empty (the JS fills it) so the tag list can be re-drawn after every
    # toggle without the server re-rendering the page — an e-ink repaint of the
    # whole article per tap would make tagging unusable.
    tag_panel = (
        (
            "<div id='reader-tag-panel' hidden>"
            "<div class='reader-tag-head'>Tags"
            "<button type='button' id='reader-tag-new' class='reader-ctl'>+ New</button>"
            "<button type='button' id='reader-tag-done' class='reader-ctl'>Done</button>"
            "</div>"
            "<form id='reader-tag-newform' hidden autocomplete='off'>"
            # Space-separated, matching the main app's tag input — the server splits
            # on whitespace, so "brand new tag" is three tags, not one. Says so,
            # because there is no autocomplete here to make that obvious.
            "<input type='text' id='reader-tag-input' placeholder='new tag (spaces split)' "
            "inputmode='text' autocapitalize='none' spellcheck='false'>"
            "<button type='submit' class='reader-ctl'>Add</button>"
            "</form>"
            "<div id='reader-tag-list'></div>"
            "</div>"
        )
        if show_saved_actions
        else ""
    )
    # NOTE: every scalar below is html.escape'd; only `article_html` is embedded
    # raw, and it is always allowlist-sanitized upstream (archived capture, live
    # readability, or stored feed content — all via html_sanitize.sanitize_html),
    # the same trust model as the existing reader-view responses. CodeQL flags
    # this as reflective-xss because our BeautifulSoup allowlist sanitizer is not
    # one of its recognized sanitizers (false positive).
    doc = (
        "<!DOCTYPE html><html lang='en'><head><meta charset='utf-8'>"
        f"<title>{head_title}</title>"
        "<meta name='viewport' content='width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no'>"
        "<meta name='robots' content='noindex'>"
        f"<meta name='csrf-token' content='{esc_csrf}'>"
        # Same vendored KaTeX as the main app's entry pane (see
        # docs/architecture/views.md "Inline LaTeX math (KaTeX)") -- Read Mode
        # has no app.js, so it needs its own copy of the CSS/font link.
        "<link rel='stylesheet' href='/static/vendor/katex-0.19.1/katex.min.css'>"
        f"<link rel='stylesheet' href='/static/reader.css?v={STATIC_ASSET_VERSION}'>"
        "</head><body>"
        "<header class='reader-bar'>"
        f"<a class='reader-ctl' href='{esc_back}' title='Back to list'>&#10005;</a>"
        f"<span class='reader-title'>{esc_title}</span>"
        "<span class='reader-pageinfo' id='reader-pageinfo'>1 / 1</span>"
        # Offline queue depth, filled by outbox.js wherever data-outbox-depth
        # appears. A queue nobody can see is how work gets lost without anyone
        # noticing; on a device that is offline by default, that is not a rare
        # case. Hidden when it is empty, which is almost always.
        "<span class='reader-outbox' data-outbox-depth hidden></span>"
        "<button class='reader-ctl' id='reader-fs-minus' type='button' title='Smaller text'>A&#8722;</button>"
        "<button class='reader-ctl' id='reader-fs-plus' type='button' title='Larger text'>A+</button>"
        f"{saved_actions}"
        f"{open_original}"
        "</header>"
        "<main id='reader-viewport'>"
        f"<div id='reader-columns' data-feed='{esc_feed}' data-entry='{esc_eid}'"
        f" data-katex-dollar-math='{1 if katex_dollar_math else 0}'>"
        f"<article id='reader-article'><h1 class='reader-headline'>{esc_title}</h1>"
        f"{reader_dateline}"
        f"<div class='reader-body'>{article_html}</div></article>"
        "</div></main>"
        f"{tag_panel}"
        f"<script>window.__READER_TAGS__={tags_json};</script>"
        f"<script>window.__READER_NAV__={nav_json};</script>"
        # Before reader.js, which calls renderMathInElement synchronously before
        # its first pagination measurement (see reader.js).
        "<script src='/static/vendor/katex-0.19.1/katex.min.js'></script>"
        "<script src='/static/vendor/katex-0.19.1/auto-render.min.js'></script>"
        f"<script src='/static/outbox.js?v={STATIC_ASSET_VERSION}'></script>"
        f"<script src='/static/reader.js?v={STATIC_ASSET_VERSION}'></script>"
        "</body></html>"
    )
    return HTMLResponse(doc, headers={"Cache-Control": "no-store"})


def _read_mode_saved_index() -> tuple[set[tuple[str, str]], dict[str, int], int, set[tuple[str, str]]]:
    """→ (inbox key set, per-feed inbox counts, archived total, filed key set).

    **The Inbox is STARRED minus Archived**, and that is a deliberate narrowing
    (2026-07-29). It briefly counted *kept* (starred OR tagged) minus archived,
    which made it 24,672 items — the whole library, not an inbox.

    The split follows from what the two signals mean. **A star is a TODO**: "I
    still have to decide what to do with this." **A tag is filing**: it is
    already sorted, and filing something is not a to-do. So tagged-but-unstarred
    entries are reachable through the tag tree, which is where you would look for
    them, and they do not sit in a queue pretending to need attention.

    *filed* (tagged minus archived) is returned alongside because the tag nodes
    must still count filed items — narrowing those to starred ones would empty
    most of the tag tree.

    Archived is its own axis (`archived_entries`), so any kept entry can be done,
    and archiving is what removes the star — the archived set is not a subset of
    the starred set.
    """
    with get_meta_connection() as conn:
        rows = conn.execute("SELECT feed_url, entry_id FROM saved_entries").fetchall()
    archived_keys = get_archived_saved_keys()
    starred: set[tuple[str, str]] = {(str(r["feed_url"]), str(r["entry_id"])) for r in rows}
    tagged = get_tagged_entry_keys(get_all_reader_feed_urls())

    inbox = starred - archived_keys
    filed = tagged - archived_keys
    feed_counts: dict[str, int] = {}
    for feed_url, _entry_id in inbox:
        feed_counts[feed_url] = feed_counts.get(feed_url, 0) + 1
    return inbox, feed_counts, len(archived_keys), filed


def _filed_tag_counts(filed: set[tuple[str, str]]) -> dict[str, int]:
    """Manual-tag counts over the *filed* set (tagged minus archived).

    Counted over filed rather than inbox items: the Inbox is starred-only now,
    so counting there would empty most of the tag tree — the majority of tagged
    entries carry no star, which is exactly why they are filed and not to-do.
    """
    if not filed:
        return {}
    prefix = MANUAL_TAG_KEY_PREFIX
    try:
        conn = sqlite3.connect(str(tenancy.reader_db_path()), timeout=5.0)
        try:
            rows = conn.execute("SELECT key, feed, id FROM entry_tags WHERE key LIKE ?", [f"{prefix}%"]).fetchall()
        finally:
            conn.close()
    except Exception:
        rows = []
    counts: dict[str, int] = {}
    for key, feed, eid in rows:
        if (str(feed), str(eid)) in filed:
            name = str(key)[len(prefix) :].strip().lower()
            if name:
                counts[name] = counts.get(name, 0) + 1
    return counts


def _read_mode_date(it: dict) -> str:
    """The date shown on a Read Mode row and under the reader's headline.

    Publish date, falling back to the received date for feeds that omit one —
    the same "effective date" the main list sorts by, so the order a sort puts
    rows in matches the dates printed on them. Both are pre-formatted upstream
    (`post_display` / `received_display`), so this adds no per-row work."""
    return str(it.get("post_display") or it.get("received_display") or "")


def _read_mode_subtitle(it: dict) -> str:
    """List subtitle: the source domain for saved-article captures (whose feed is
    the synthetic 'Saved Articles'), otherwise the originating feed's title."""
    if str(it.get("feed_url")) == saved_articles_service.SAVED_FEED_URL:
        host = urlparse(str(it.get("link") or "")).netloc
        return host[4:] if host.startswith("www.") else host
    return str(it.get("feed_title") or "")


# Read Mode sort options: label -> (sort_by, sort_dir) for resolve_reader_backlog.
# "Oldest first" exists because that is how a comic backlog is actually read —
# the reader pinned newest-first and offered no way to change it.
_READ_SORTS: dict[str, tuple[str, str]] = {
    "new": ("post", "desc"),
    "old": ("post", "asc"),
    "recent": ("received", "desc"),
    "starred": ("starred", "desc"),
}
_READ_SORT_DEFAULT = "new"
# The Inbox is a to-do pile, so its natural order is when you added something to
# it, not when the article was published. Every other node keeps the global
# default — see _read_sort_for_node.
_READ_SORT_INBOX_DEFAULT = "starred"
_READ_SORT_LABELS: dict[str, str] = {
    "new": "Newest",
    "old": "Oldest",
    "recent": "Received",
    "starred": "Recently starred",
}


def _read_is_inbox_node(folder_id: int | None, tag: str | None, archived: bool, q: str | None, scope: str, root_id: int | None) -> bool:
    """Is this the Inbox node? (saved scope, ROOT folder, no tag/archive/search)

    The root check is the whole point and was missing: every saved folder node
    links with its own id (`_read_browse_href(fid, ...)`), so `folder_id is not
    None` matched all of them. Opening any saved folder in Read Mode then
    inherited the Inbox's semantics — starred-only `kept_scope` and the
    most-recently-starred default order — instead of the saved defaults.
    Caught by Sourcery on the stack review; `on_all` beside it already compared
    against root_id correctly.
    """
    if scope == "feeds" or tag or archived or q:
        return False
    return folder_id is not None and root_id is not None and folder_id == root_id


def _read_sort_for_node(sort: str | None, *, is_inbox: bool) -> str:
    """The effective order for a node: an explicit choice, else the node default.

    Only the Inbox differs — it is a to-do pile, so it opens most-recently-
    starred. Everywhere else that order would be noise (most items never carried
    a star at all), so the global newest-first default stands."""
    if sort in _READ_SORTS:
        return sort
    return _READ_SORT_INBOX_DEFAULT if is_inbox else _READ_SORT_DEFAULT


def _read_mode_sort_options(current: str, href_for: Callable[[str], str], *, include_starred: bool = True) -> list[dict]:
    """The sort switcher for the browse pane — one link per order, current marked.

    "Recently starred" is offered only where it means something: the saved scope,
    where every row has a star date. In the feeds scope most entries were never
    starred, so the order would be arbitrary."""
    return [
        {"key": key, "label": _READ_SORT_LABELS[key], "href": href_for(key), "active": key == current}
        for key in _READ_SORTS
        if not (key == "starred" and not include_starred)
    ]


def _read_mode_scope_tabs(current_scope: str) -> list[dict]:
    """The Saved / Feeds switcher shown at the top of the Read Mode tree so you
    can flip between the two scopes without leaving Read Mode.

    Saved's href carries a harmless ``home=1`` (ignored by the route) rather
    than being bare ``/read`` — same trick as the main app's wordmark
    (``/?home=1``): the resume-restore script only redirects a *bare* URL, so
    this is the one link that reliably reaches the true landing instead of
    bouncing back into whatever was last read."""
    return [
        {"label": "Feeds", "glyph": "☰", "href": "/read?scope=feeds", "active": current_scope == "feeds"},
        {"label": "Saved", "glyph": "★", "href": "/read?home=1", "active": current_scope == "saved"},
    ]


def _read_mode_search_fields(
    *,
    scope: str,
    folder_id: int | None = None,
    list_feed_url: str | None = None,
    tag: str | None = None,
    archived: bool = False,
) -> list[dict]:
    """Hidden fields the Read Mode search form must carry to stay inside the
    node you searched from.

    The form used to post only `scope`, so searching from a folder, feed, tag,
    or Archive silently threw that selection away and searched everything.
    """
    fields = [{"name": "scope", "value": scope}] if scope and scope != "saved" else []
    if folder_id is not None:
        fields.append({"name": "folder_id", "value": str(folder_id)})
    if list_feed_url:
        fields.append({"name": "list_feed_url", "value": list_feed_url})
    if tag:
        fields.append({"name": "tag", "value": tag})
    if archived:
        fields.append({"name": "archived", "value": "1"})
    return fields


def _read_clear_search_href(fields: list[dict]) -> str:
    """Same node, no query — what the Clear control returns you to."""
    params = urlencode([(f["name"], f["value"]) for f in fields])
    return "/read" + (f"?{params}" if params else "")


def _build_feeds_mode_context(
    request: Request,
    *,
    folder_id: int | None,
    list_feed_url: str | None,
    tag: str | None,
    q: str | None,
    items: list[dict],
    node_selected: bool = True,
) -> dict:
    """Read Mode context for the FEEDS scope: a feeds tree (All + folders that
    expand to their individual feeds + manual-tag buckets) and the unread item
    list for the selected node. Mirrors the regular view's navigation."""
    with get_meta_connection() as conn:
        snapshot = get_meta_structure_snapshot(conn)
    root_id = cast(int, snapshot["root_id"])
    all_reader_feed_urls = get_all_reader_feed_urls()
    raw_folder_rows = cast("list[dict]", snapshot["raw_folder_rows"])
    direct = cast("dict[int, list[str]]", snapshot["direct_feed_urls_by_folder"])
    unread_by_feed = get_unread_counts_by_feed()
    unread_by_folder = get_unread_counts_by_folder(raw_folder_rows, unread_by_feed, direct)
    title_map = get_feed_title_map()

    def _feed_nodes(fid: int, feed_urls: list[str]) -> list[dict]:
        nodes = [
            {
                "label": title_map.get(furl, furl),
                "href": _read_browse_href(fid, None, False, None, "feeds", list_feed_url=furl),
                "count": unread_by_feed.get(furl, 0),
                "active": (list_feed_url == furl),
            }
            for furl in feed_urls
            if unread_by_feed.get(furl, 0)
        ]
        nodes.sort(key=lambda n: str(n["label"]).lower())
        return nodes

    on_all = folder_id == root_id and not list_feed_url and not tag and not q
    folder_nodes: list[dict] = [
        {
            # Empty glyph: "All" is plain navigation, so no expand arrow — but the
            # spacer keeps its label aligned with the folder rows' caret+label.
            "label": "All",
            "glyph": "",
            "href": _read_browse_href(root_id, None, False, None, "feeds"),
            "count": unread_by_folder.get(root_id, 0),
            "active": on_all,
        }
    ]
    for row in raw_folder_rows:
        fid = int(row["id"])
        if fid == root_id:
            continue
        c = unread_by_folder.get(fid, 0)
        if not c:
            continue
        feeds = _feed_nodes(fid, direct.get(fid, []))
        folder_nodes.append(
            {
                "label": str(row["name"]),
                "glyph": "▸",
                "href": _read_browse_href(fid, None, False, None, "feeds"),
                "count": c,
                "active": (not tag and not list_feed_url and folder_id == fid),
                "feeds": feeds,
                "open": any(f["active"] for f in feeds),  # keep expanded if a feed is active
            }
        )

    # Manual-tag buckets — the regular Tags (all entries, NOT saved-only).
    tag_nodes = [
        {
            "label": "#" + str(tr["name"]),
            "glyph": "",
            "href": _read_browse_href(None, str(tr["name"]), False, None, "feeds"),
            "count": int(tr["count"]),
            "active": (tag == str(tr["name"])),
        }
        for tr in get_tag_counts_for_feeds(set(all_reader_feed_urls))
    ]

    if not node_selected:
        selected_label = "Feeds"
    elif q:
        selected_label = f"Search: “{q}”"
    elif list_feed_url:
        selected_label = title_map.get(list_feed_url, list_feed_url)
    elif tag:
        selected_label = "#" + tag
    else:
        selected_label = next((n["label"] for n in folder_nodes if n["active"]), "All")

    list_items = [
        {
            "title": str(it.get("title") or it.get("link") or "(untitled)"),
            "title_html": html_sanitize.sanitize_inline_title(str(it.get("title") or it.get("link") or "(untitled)")),
            "subtitle": str(it.get("feed_title") or ""),
            "read": bool(it.get("read", False)),
            "href": _reader_href(
                str(it["feed_url"]),
                str(it["id"]),
                folder_id=folder_id,
                tag=tag,
                archived=False,
                q=q,
                scope="feeds",
                list_feed_url=list_feed_url,
            ),
        }
        for it in items
    ]

    _feeds_search_fields = _read_mode_search_fields(
        scope="feeds",
        folder_id=folder_id,
        list_feed_url=list_feed_url,
        tag=tag,
    )
    return {
        "scope_tabs": _read_mode_scope_tabs("feeds"),
        "folder_nodes": folder_nodes,
        "tag_nodes": tag_nodes,
        "archive_node": None,
        "list_items": list_items,
        "selected_label": selected_label,
        "node_selected": node_selected,
        "search_query": q or "",
        "search_fields": _feeds_search_fields,
        "read_clear_search_href": _read_clear_search_href(_feeds_search_fields),
        "tags_open": bool(tag),
        "scope": "feeds",
        "exit_href": "/?full=1",
        "static_asset_version": STATIC_ASSET_VERSION,
    }


def _build_read_mode_context(
    request: Request,
    *,
    folder_id: int | None,
    tag: str | None,
    list_feed_url: str | None = None,
    archived: bool,
    q: str | None,
    items: list[dict],
    node_selected: bool = True,
    sort: str = _READ_SORT_DEFAULT,
    resume_sort: str | None = None,
    all_saved: bool = False,
    confirm_delete_tag: str | None = None,
) -> dict:
    """Assemble the Read Mode 2-pane context: the simplified saved tree (folders
    + tag buckets + Archive, pinned) and the item list for the selected node.
    All tree counts are the non-archived (inbox) totals. When ``node_selected``
    is false (a bare /read landing) the right pane stays empty — we don't load
    the whole backlog just to open the app."""
    with get_meta_connection() as conn:
        snapshot = get_meta_structure_snapshot(conn)
    root_id = cast(int, snapshot["root_id"])
    all_feed_urls = cast(set[str], snapshot["all_feed_urls"])
    raw_folder_rows = cast("list[dict]", snapshot["raw_folder_rows"])
    all_reader_feed_urls = get_all_reader_feed_urls()

    folder_feed_urls_by_id = dict(cast("dict[int, set[str]]", snapshot["folder_feed_urls_by_id"]))
    folder_feed_urls_by_id[root_id] = set(all_reader_feed_urls)
    folder_feed_urls_by_id[UNCATEGORIZED_FOLDER_ID] = all_reader_feed_urls - all_feed_urls

    inbox, _feed_inbox_counts, archived_count, filed = _read_mode_saved_index()

    # Folder nodes count KEPT (starred or tagged), not starred alone. The badge
    # has to agree with what opening the folder lists, and a saved folder lists
    # kept — only the Inbox node is narrowed to stars. Counting stars here made
    # Booze read 11 against the main app's 67, and hid outright the three folders
    # whose saved items are all tagged and none starred.
    _feed_kept_counts: dict[str, int] = {}
    for _f, _e in inbox | filed:
        _feed_kept_counts[_f] = _feed_kept_counts.get(_f, 0) + 1

    def _folder_kept_count(fid: int) -> int:
        feeds = folder_feed_urls_by_id.get(fid, set())
        return sum(c for f, c in _feed_kept_counts.items() if f in feeds)

    on_all = folder_id == root_id and not tag and not archived and not q and not all_saved
    # Links *out* of the Inbox must not carry its most-recently-starred order —
    # that ordering is meaningless in a folder where most items were never
    # starred. Hand back the order you had before entering the Inbox instead.
    outbound_sort = (resume_sort or _READ_SORT_DEFAULT) if on_all else sort
    folder_nodes: list[dict] = [
        {
            # "Inbox", not "All" — this node counts and lists `inbox`, which is
            # STARRED minus archived, not everything saved. Calling it All was
            # survivable while Archive held 23 items; now that archiving is half the
            # triage model it is just wrong, and it sits directly above a node named
            # Archive holding what it excludes. Filed (tagged) items live in the tag
            # tree. (The feeds scope keeps "All": no archive axis there.)
            # Empty glyph: plain navigation, so no expand arrow — but the spacer
            # keeps its label aligned with the folder rows.
            "label": "Inbox",
            "glyph": "",
            # No sort= (the Inbox picks its own default), but resume_sort carries the
            # order you were using so stepping out of the Inbox restores it.
            "href": _read_browse_href(root_id, None, False, None, resume_sort=(sort if not on_all else resume_sort)),
            "count": len(inbox),
            "active": on_all,
        },
        {
            # Everything kept minus archived — what the main app's Saved view shows.
            # The Inbox is deliberately narrower (starred only), so without this node
            # the ~15k tagged-but-unstarred items would exist in one mode and not the
            # other. They are all reachable under Tags; this is the flat view.
            "label": "All Saved",
            "glyph": "",
            "href": _read_browse_href(root_id, None, False, None, sort=outbound_sort, kept_all=True),
            "count": len(inbox | filed),
            "active": all_saved,
        },
    ]
    for row in raw_folder_rows:
        fid = int(row["id"])
        if fid == root_id:
            continue
        c = _folder_kept_count(fid)
        if not c:
            continue
        folder_nodes.append(
            {
                "label": str(row["name"]),
                "glyph": "▸",  # ▸
                "href": _read_browse_href(fid, None, False, None, sort=outbound_sort),
                "count": c,
                "active": (not archived and not tag and folder_id == fid),
            }
        )
    uncat = _folder_kept_count(UNCATEGORIZED_FOLDER_ID)
    if uncat:
        folder_nodes.append(
            {
                "label": "Uncategorized",
                "glyph": "▸",  # ▸
                "href": _read_browse_href(UNCATEGORIZED_FOLDER_ID, None, False, None, sort=outbound_sort),
                "count": uncat,
                "active": (not archived and not tag and folder_id == UNCATEGORIZED_FOLDER_ID),
            }
        )

    # Manual-tag buckets restricted to the inbox: only tags on non-archived saved
    # items, counted over them. Tucked in a collapsed <details> (heavy taggers
    # have dozens); clicking narrows the inbox to that tag.
    inbox_tag_counts = _filed_tag_counts(filed)
    tag_nodes = [
        {
            "label": "#" + name,
            "glyph": "",
            "href": _read_browse_href(None, name, False, None, sort=outbound_sort),
            "count": inbox_tag_counts[name],
            "active": (not archived and tag == name),
        }
        for name in sorted(inbox_tag_counts)
    ]

    if not node_selected:
        selected_label = "Saved"
    elif all_saved:
        selected_label = "All Saved"
    elif archived:
        selected_label = "Archive"
    elif q:
        selected_label = f"Search: “{q}”"
    elif tag:
        selected_label = "#" + tag
    else:
        selected_label = next((n["label"] for n in folder_nodes if n["active"]), "Inbox")

    list_items = [
        {
            "title": str(it.get("title") or it.get("link") or "(untitled)"),
            "title_html": html_sanitize.sanitize_inline_title(str(it.get("title") or it.get("link") or "(untitled)")),
            "subtitle": _read_mode_subtitle(it),
            "date": _read_mode_date(it),
            "read": bool(it.get("read", False)),
            "href": _reader_href(
                str(it["feed_url"]), str(it["id"]), folder_id=folder_id, tag=tag, archived=archived, q=q, sort=sort, kept_all=all_saved
            ),
        }
        for it in items
    ]

    _saved_search_fields = _read_mode_search_fields(
        scope="saved",
        folder_id=folder_id,
        tag=tag,
        archived=archived,
    )
    # Bulk actions for the node you drilled into. Read Mode has no right-click and
    # long-press offers only text selection, so these are visible buttons with big
    # tap targets — the same plain-navigation model as the Sort switcher, which
    # means they work with no JS and cost one e-ink repaint each.
    node_actions = None
    if node_selected and not archived and (tag or list_feed_url):
        _scope_stars = len(_scope_starred_keys(folder_id, list_feed_url, tag))
        node_actions = {
            "tag": tag or "",
            "folder_id": folder_id,
            "list_feed_url": list_feed_url or "",
            "star_count": _scope_stars,
            # Deleting a tag is irreversible, so it takes two taps rather than a
            # modal: the first re-renders this row with the count spelled out. A
            # browser confirm() is an awkward thing to hit on that WebView.
            "confirm_delete_tag": bool(tag) and confirm_delete_tag == "1",
            "confirm_href": _read_browse_href(folder_id, tag, False, None, sort=sort, list_feed_url=list_feed_url)
            + (
                ("&" if "?" in _read_browse_href(folder_id, tag, False, None, sort=sort, list_feed_url=list_feed_url) else "?")
                + "confirm_delete_tag=1"
            ),
            "cancel_href": _read_browse_href(folder_id, tag, False, None, sort=sort, list_feed_url=list_feed_url),
        }
    return {
        "node_actions": node_actions,
        # request is None in unit tests that build the context directly; the
        # actions row simply renders without a token there.
        "csrf_token": _csrf_token_for(request) if request is not None else "",
        "sort_options": _read_mode_sort_options(
            sort,
            lambda key: _read_browse_href(folder_id, tag, archived, q, sort=key, resume_sort=resume_sort, kept_all=all_saved),
        ),
        "scope_tabs": _read_mode_scope_tabs("saved"),
        "folder_nodes": folder_nodes,
        "tag_nodes": tag_nodes,
        "archive_node": {
            "label": "Archive",
            "glyph": "▤",  # ▤
            "href": _read_browse_href(None, None, True, None, sort=outbound_sort),
            "count": archived_count,
            "active": archived,
        },
        "list_items": list_items,
        "selected_label": selected_label,
        "node_selected": node_selected,
        "search_query": q or "",
        "search_fields": _saved_search_fields,
        "read_clear_search_href": _read_clear_search_href(_saved_search_fields),
        # Open by default. Tags are no longer a side-bucket of the Inbox — the
        # Inbox is starred-only, so filed items are reachable *only* here, and a
        # collapsed <details> made them look absent (the "I don't see #inbox in
        # eInk" report: it was there, behind the disclosure).
        "tags_open": True,
        "scope": "saved",
        "exit_href": "/",
        "static_asset_version": STATIC_ASSET_VERSION,
    }


# Distinct User-Agents seen on /read, logged once each (capped) so we can learn
# the e-ink devices' UA for auto-detect (it isn't in access logs).
#
# Captured 2026-07-29 from Josh's Manta:
#   Mozilla/5.0 (Linux; Android 11; Supernote Nomad Build/RQ2A.210505.003; wv)
#   AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/96.0.4664.92 …
#
# ⚠ It says **Nomad on a Manta** — the model is hardcoded in Supernote's browser,
# not detected, and the Build fingerprint is stock Android 11. So an auto-detect
# must match "Supernote", never the model. `wv` also matters: this is an Android
# WebView, so storage persistence and service-worker availability are the host
# app's call, not the web platform's — see the offline-reading note in Plan.md.
_READ_MODE_UA_SEEN: set[str] = set()


# --- Account / user management ---


def _current_web_user(request: Request) -> str | None:
    """The logged-in web session's user_id, or None."""
    if not request.session.get("authenticated"):
        return None
    uid = request.session.get("user_id")
    return uid if uid and tenancy.is_valid_user_id(uid) else None


def _current_web_username(request: Request) -> str | None:
    """Display name (mutable username) for the current session user, or None."""
    uid = _current_web_user(request)
    if not uid:
        return None
    row = user_store.get_by_id(uid)
    return row["username"] if row else None


def _is_web_admin(user_id: str | None) -> bool:
    if not user_id:
        return False
    row = user_store.get_by_id(user_id)
    return bool(row and row["is_admin"] and not row["disabled"])


def _dir_bytes(*paths: Path) -> int:
    """Total bytes for a set of DB files including their -wal/-shm/.search sidecars."""
    total = 0
    for p in paths:
        for f in p.parent.glob(p.name + "*"):
            try:
                total += f.stat().st_size
            except OSError:
                pass
    return total


# Log levels ranked so the Logs tab's filter can select a minimum severity.
_LOG_LEVEL_RANK = {"DEBUG": 10, "INFO": 20, "WARNING": 30, "ERROR": 40, "CRITICAL": 50}
_LOG_LINE_RE = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}[,.]\d+ (\w+) ")
# The ``[tag]`` that opens a record's message, right after "<timestamp> <LEVEL> <logger>: ". Anchored so a bracketed word quoted
# later in an untagged line (a URL or title) isn't mistaken for a subsystem tag.
_LOG_TAG_RE = re.compile(r"^\S+ \S+ \w+ [^\s:]+: \[([a-z][a-z0-9 _-]*)\]", re.IGNORECASE)

# Admin → Logs "Category" filter: groups the bracketed ``[subsystem]`` tags that log records carry. Exact tags
# match whole; a trailing "*" matches by prefix and a leading "*" by suffix. Untagged records (HTTP access
# lines etc.) and unlisted tags fall under "other".
_LOG_CATEGORIES: dict[str, tuple[str, ...]] = {
    "automations": ("automation", "on-star", "tag-filter", "autofile", "*-auto", "curation"),
    "maintenance": (
        "maintenance",
        "purge",
        "dedup",
        "saved-dedup",
        "guid-churn*",
        "reprobe",
        "auto-refetch",
        "hide-locked-comics",
        "perf",
        "admin-vacuum",
        "archive-old-stars",
        "startup",
        "shutdown",
    ),
    "refresh": (
        "refresh",
        "scheduler",
        "websub",
        "scraper",
        "full-fetch",
        "page-fetch",
        "re-fetch",
        "refetch-batch",
        "canonical-link",
        "page-topics",
        "extract",
        "prefetch-diag",
    ),
    "youtube": ("yt-*", "youtube*", "hide-unpremiered", "backfill-hide-shorts"),
    "integrations": (
        "deviantart",
        "devto",
        "reddit",
        "inoreader*",
        "pinterest-oauth",
        "quire*",
        "freshrss-import",
        "bsky",
        "greader",
        "fever",
        "takeout",
        "ttrss-import",
        "miniflux-import",
        "webtoons",
        "webcomic",
        "tapas",
        "instapaper*",
        "email*",
        "webhook*",
    ),
    "images": ("img-cache", "feed-thumb", "entry-thumb"),
}


LOG_CATEGORY_NAMES = frozenset({*_LOG_CATEGORIES, "other"})


def _log_categories(line: str) -> set[str]:
    """Categories of a log record (see ``_LOG_CATEGORIES``), from the first ``[tag]`` after its header. A record can
    belong to several (a YouTube playlist automation is both "automations" and "youtube"); YouTube API calls
    logged by httpx carry no tag and are recognized by URL. Records matching nothing are "other"."""
    found: set[str] = set()
    tag_m = _LOG_TAG_RE.match(line)
    if tag_m:
        tag = tag_m.group(1).lower()
        for category, patterns in _LOG_CATEGORIES.items():
            for pat in patterns:
                if pat.endswith("*"):
                    hit = tag.startswith(pat[:-1])
                elif pat.startswith("*"):
                    hit = tag.endswith(pat[1:])
                else:
                    hit = tag == pat
                if hit:
                    found.add(category)
                    break
    if "googleapis.com/youtube/" in line:
        found.add("youtube")
    return found or {"other"}


def _log_line_dt(line: str) -> datetime | None:
    """Parse a log record's leading ``YYYY-MM-DD HH:MM:SS`` timestamp, or None
    for a continuation line (traceback etc.)."""
    if not _LOG_LINE_RE.match(line):
        return None
    try:
        return datetime.strptime(line[:19], "%Y-%m-%d %H:%M:%S")
    except ValueError, IndexError:
        return None


def _parse_local_ts(value: str) -> tuple[datetime | None, bool]:
    """Parse a datetime-local / space-separated timestamp. Returns
    (dt, had_seconds); dt is None if unparseable."""
    value = (value or "").strip()
    if not value:
        return None, False
    for fmt in ("%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S"):
        try:
            return datetime.strptime(value, fmt), True
        except ValueError:
            pass
    for fmt in ("%Y-%m-%dT%H:%M", "%Y-%m-%d %H:%M"):
        try:
            return datetime.strptime(value, fmt), False
        except ValueError:
            pass
    return None, False


def _read_log_tail(
    max_lines: int, min_level: str, since: datetime | None = None, until: datetime | None = None, category: str = ""
) -> tuple[list[str], bool, bool]:
    """Return (lines, available, truncated). Reads the current lectio.log, keeps
    records at or above ``min_level``, within [``since``, ``until``] and (when
    ``category`` is set) of that category — continuation lines like tracebacks stay attached to their record — and
    returns the last ``max_lines`` of the match. ``truncated`` is True when the
    cap dropped older matching records. available=False when file logging isn't
    configured."""
    log_dir_str = os.getenv("LECTIO_LOG_DIR", "").strip()
    if not log_dir_str:
        return [], False, False
    path = Path(log_dir_str) / "lectio.log"
    if not path.exists():
        return [], True, False
    threshold = _LOG_LEVEL_RANK.get(min_level.upper(), 0)
    category = category.strip().lower()
    if category == "all":
        category = ""
    try:
        raw = path.read_text(encoding="utf-8", errors="replace").splitlines()
    except OSError:
        return [], True, False
    if threshold <= 0 and since is None and until is None and not category:
        return raw[-max_lines:], True, len(raw) > max_lines
    kept: list[str] = []
    include_current = False
    for line in raw:
        m = _LOG_LINE_RE.match(line)
        if m:
            include_current = _LOG_LEVEL_RANK.get(m.group(1), 0) >= threshold
            if include_current and (since is not None or until is not None):
                dt = _log_line_dt(line)
                if dt is not None and ((since is not None and dt < since) or (until is not None and dt > until)):
                    include_current = False
            if include_current and category and category not in _log_categories(line):
                include_current = False
            if include_current:
                kept.append(line)
        elif include_current:
            kept.append(line)  # continuation (traceback etc.) of an included record
    return kept[-max_lines:], True, len(kept) > max_lines


def _home_inner(
    request: Request,
    folder_id: int | None = None,
    list_feed_url: str | None = None,
    tag: str | None = None,
    feed_tag: str | None = None,
    sort_by: str | None = None,
    sort_dir: str | None = None,
    read_filter: str | None = None,
    star_only: str | None = None,
    saved_home: int | None = None,
    home: int | None = None,
    resume_read_filter: str | None = None,
    feed_url: str | None = None,
    entry_id: str | None = None,
    q: str | None = None,
    message: str | None = None,
    no_rss_url: str | None = None,
    force_url: str | None = None,
    chunk: int | None = None,
    chunk_delta: str | None = None,
    chunk_size: int | None = None,
    filter_term: str | None = None,
    subscribe: str | None = None,
    kept: str | None = None,
):
    # Allow client to suggest a preferred read_filter via header for SPA/AJAX calls.
    # Use it only when no explicit `read_filter` was supplied in the URL/form.
    # Explicit query/form state must win to avoid client preference overriding
    # an intentional filter selection during navigation.
    header_rf = request.headers.get("X-Lectio-Read-Filter")
    if read_filter is None and header_rf and not normalize_star_only(star_only):
        read_filter = header_rf

    # Only use `resume_read_filter` as a fallback when no explicit
    # `read_filter` was supplied.
    if read_filter is None and resume_read_filter is not None:
        read_filter = resume_read_filter

    # If still no explicit value, fall back to the remembered preference — but
    # only for Feeds browsing. The Saved view always defaults to All: saved
    # items are usually already read (Archive is their done-axis), so a
    # remembered Unread filter made folders look empty while their badges
    # showed the total. The Unread pill still narrows explicitly per-visit.
    if read_filter is None:
        if normalize_star_only(star_only):
            read_filter = "all"
        else:
            cookie_rf = request.cookies.get("lectio_read_filter")
            if cookie_rf:
                read_filter = cookie_rf

    start_req = time.perf_counter()
    _t = time.perf_counter()

    def _tick(label: str) -> None:
        nonlocal _t
        ms = int((time.perf_counter() - _t) * 1000)
        if ms >= 50:
            LOGGER.info("[perf]   meta.%s=%dms", label, ms)
        _t = time.perf_counter()

    with get_meta_connection() as conn:
        _tick("connect")
        # Per-scope: Feeds and Saved keep their own remembered order.
        _sort_by_key, _sort_dir_key = sort_setting_keys(normalize_star_only(star_only))
        # "starred" (order by saved_entries.saved_at) means something only in the
        # Saved scope — in Feeds most entries were never starred, so the order
        # would be arbitrary. Gating it on the scope rather than blessing it
        # globally is what keeps it out of the Feeds view, where it once leaked
        # in and left the toolbar reading "Published newest" over a star-ordered
        # list. It has to be allowed on the READ back too, or a remembered
        # "starred" would be silently rewritten to the default on every load —
        # the preference quietly destroying itself.
        _allow_starred_sort = normalize_star_only(star_only)
        preferred_sort_by = normalize_sort_by(get_setting(conn, _sort_by_key), allow_starred=_allow_starred_sort)
        preferred_sort_dir = normalize_sort_dir(get_setting(conn, _sort_dir_key))
        problematic_feeds_last_viewed_at = parse_epoch_setting(get_setting(conn, PROBLEMATIC_FEEDS_LAST_VIEWED_AT_SETTING_KEY))
        # The Saved Inbox is the whole star pile — every star, however it was
        # made (by hand, by a save_article rule, by the browser extension) —
        # and a to-do pile's natural order is when you added to it. It opens
        # most-recently-starred, which is a NODE DEFAULT, not a URL param: the
        # index persists any explicit sort_by it is handed, and a default that
        # persisted would overwrite the order you chose for the rest of Saved.
        inbox_view = _allow_starred_sort and kept == "starred"
        _node_default_sort_by = "starred" if inbox_view else preferred_sort_by
        _node_default_sort_dir = "desc" if inbox_view else preferred_sort_dir
        # The star order belongs to the Inbox node, and only to it. Two guards,
        # because the templates hand `sort_by` around in ~14 places (post links,
        # search, action forms) and one of them forgetting `kept=starred` must
        # not be able to turn the whole Saved scope star-ordered:
        #   * honored only ON the Inbox — a stray ?sort_by=starred elsewhere
        #     falls back to that scope's own remembered order;
        #   * never PERSISTED — it is a node default, and persisting it would
        #     overwrite the order chosen for the rest of Saved. This is the
        #     exact shape of the bug that once left the toolbar reading
        #     "Published newest" over a star-ordered list.
        selected_sort_by = normalize_sort_by(sort_by or _node_default_sort_by, allow_starred=_allow_starred_sort)
        if selected_sort_by == "starred" and not inbox_view:
            selected_sort_by = normalize_sort_by(preferred_sort_by)
        selected_sort_dir = normalize_sort_dir(sort_dir or _node_default_sort_dir)
        # Persist only an EXPLICIT choice from the sort menu. Re-saving on every
        # plain load meant any stored value normalize_sort_by did not recognize
        # was silently replaced by the default — the remembered preference
        # quietly destroying itself, with no way to tell it had happened.
        # Neither half of the Inbox's order is persistable. Blocking only the
        # KEY let its direction through: visiting the Inbox (desc) flipped the
        # remembered Saved direction, so leaving it turned "All" from
        # oldest-first to newest-first. A sort is a pair, and a node default has
        # to be inert as a pair.
        # A sort_by requesting a star-only order ("starred"/"size") from
        # OUTSIDE the star scope isn't a real choice for the scope it landed
        # in — it's exactly what a stale Saved-scope link (or a stale
        # address-bar URL, force-refreshed) carries. normalize_sort_by
        # already falls the KEY back to the default safely, but that default
        # is a hardcoded "post", not this scope's own actual remembered
        # value — and sort_dir is valid in any scope, so it sails through
        # untouched. Either half persisting here would silently overwrite
        # what this scope had remembered with values that were never a
        # genuine choice for it. Reported live 2026-09-13 as "my FEEDS sort
        # keeps getting reset to Pub new": browsing Saved sorted by size,
        # then clicking a Feeds-tree link (which index.html no longer builds
        # this way — see _feeds_tree_sq) silently flipped the Feeds scope's
        # own remembered direction from asc ("Pub old") to desc ("Pub new").
        _sort_from_wrong_scope = bool(sort_by) and sort_by in {"starred", "size"} and not _allow_starred_sort
        if selected_sort_by != "starred" and not _sort_from_wrong_scope:
            if sort_by:
                set_setting(conn, _sort_by_key, selected_sort_by)
            if sort_dir:
                set_setting(conn, _sort_dir_key, selected_sort_dir)
        _tick("settings")

        snapshot = get_meta_structure_snapshot(conn)
        raw_folder_rows = cast(list[dict], snapshot["raw_folder_rows"])
        direct_feed_urls_by_folder = cast(dict[int, list[str]], snapshot["direct_feed_urls_by_folder"])
        folder_options = cast(list[FolderOption], snapshot["folder_options"])
        all_feed_urls = cast(set[str], snapshot["all_feed_urls"])
        root_id = cast(int, snapshot["root_id"])
        folder_feed_urls_by_id = cast(dict[int, set[str]], snapshot["folder_feed_urls_by_id"])
        selected_folder_id = folder_id or root_id
        _tick("structure_snapshot")

        # Derive the virtual "Uncategorized" folder: feeds the reader knows about
        # that live in no folder_feeds row. Copy the snapshot's per-folder maps
        # (shared with the cached snapshot) before adding derived keys so we never
        # mutate cross-request cache state. Also widen "All Feeds" (root) to cover
        # every reader feed, not just foldered ones, so orphan feeds and their
        # unreads are reachable from the top of the tree.
        all_reader_feed_urls = get_all_reader_feed_urls()
        uncategorized_feed_urls = all_reader_feed_urls - all_feed_urls
        # The local Saved Articles feed is surfaced by the Saved sidebar view,
        # not as a subscription: it must never show — or be browsable — as a
        # feed in Feeds mode, root or Uncategorized alike. Its reachability
        # from the Saved sidebar's OWN "Uncategorized" grouping is served
        # below by the star_only-gated re-inclusion (mirroring root), not by
        # keeping it in this shared dict — that leaked it into a plain Feeds
        # browse of Uncategorized, showing its whole backlog as if it were an
        # ordinary orphan feed's entries (found 2026-08-27).
        _uncat_display_urls = uncategorized_feed_urls - {saved_articles_service.SAVED_FEED_URL}
        folder_feed_urls_by_id = dict(folder_feed_urls_by_id)
        direct_feed_urls_by_folder = dict(direct_feed_urls_by_folder)
        folder_feed_urls_by_id[root_id] = all_feed_urls | _uncat_display_urls
        folder_feed_urls_by_id[UNCATEGORIZED_FOLDER_ID] = _uncat_display_urls
        direct_feed_urls_by_folder[UNCATEGORIZED_FOLDER_ID] = sorted(_uncat_display_urls)
        _tick("uncategorized_derive")

        unread_counts_by_feed = get_unread_counts_by_feed()
        _tick("unread_counts")
        disabled_feed_urls = get_disabled_feed_urls(conn)
        # Exclude disabled feeds from unread counts so folder badges stay clean.
        active_unread_counts_by_feed = {url: count for url, count in unread_counts_by_feed.items() if url not in disabled_feed_urls}
        unread_counts_by_folder = get_unread_counts_by_folder(
            raw_folder_rows,
            active_unread_counts_by_feed,
            direct_feed_urls_by_folder,
        )
        # The virtual Uncategorized folder isn't in raw_folder_rows, so count it
        # here and fold it into the root ("All Feeds") total.
        uncategorized_unread = sum(active_unread_counts_by_feed.get(url, 0) for url in _uncat_display_urls)
        unread_counts_by_folder[UNCATEGORIZED_FOLDER_ID] = uncategorized_unread
        unread_counts_by_folder[root_id] = unread_counts_by_folder.get(root_id, 0) + uncategorized_unread
        _tick("counts_by_folder")
        folder_rows = []
        for row in raw_folder_rows:
            folder_dict = dict(row)
            folder_dict["unread_count"] = unread_counts_by_folder.get(int(row["id"]), 0)
            folder_rows.append(folder_dict)
        # Append the virtual Uncategorized folder last so it pins to the bottom of
        # the tree. Only when it actually holds feeds. `virtual` flags the template
        # and context menu to suppress edit affordances (rename/delete/props).
        if uncategorized_feed_urls:
            folder_rows.append(
                {
                    "id": UNCATEGORIZED_FOLDER_ID,
                    "name": UNCATEGORIZED_FOLDER_NAME,
                    "cadence_minutes": None,
                    "depth": 1,
                    "path": UNCATEGORIZED_FOLDER_NAME,
                    "feed_count": len(_uncat_display_urls),
                    "unread_count": uncategorized_unread,
                    "virtual": True,
                    # Saved-only membership (just lectio:saved unfoldered): the
                    # feeds tree hides the row; the Saved sublist still shows it.
                    "has_display_feeds": bool(_uncat_display_urls),
                }
            )
        global_note = get_setting(conn, GLOBAL_NOTE_SETTING_KEY) or ""
        email_to_default = get_setting(conn, EMAIL_TO_SETTING_KEY) or "" if is_email_configured() else ""
        highlight_rules = get_highlight_keywords(conn)
        profile_name = get_setting(conn, PROFILE_NAME_SETTING_KEY) or ""
        profile_email = get_setting(conn, PROFILE_EMAIL_SETTING_KEY) or ""
        _profile_lower = profile_email.lower()
        _avatar_hash = hashlib.md5(_profile_lower.strip().encode()).hexdigest() if _profile_lower.strip() else ""
        profile_avatar_url = (
            f"https://www.gravatar.com/avatar/{_avatar_hash}?d=identicon&s=128"
            if _avatar_hash
            else "https://www.gravatar.com/avatar/?d=identicon&s=128"
        )
        email_contacts = [c for c in get_email_contacts(conn) if c["address"].lower() != _profile_lower]
        email_bcc = get_setting(conn, EMAIL_BCC_SETTING_KEY) or ""
        youtube_sync_last_at = get_setting(conn, YOUTUBE_SYNC_LAST_AT_KEY) or ""
        youtube_sync_last_result = get_setting(conn, YOUTUBE_SYNC_LAST_RESULT_KEY) or ""
        # Build inactive feed list (feed_url + folder membership).
        inactive_feed_rows = conn.execute(
            """
            SELECT df.feed_url, df.disabled_at, df.reason, ff.folder_id, f.name AS folder_name
            FROM disabled_feeds df
            LEFT JOIN folder_feeds ff ON ff.feed_url = df.feed_url
            LEFT JOIN folders f ON f.id = ff.folder_id
            ORDER BY df.disabled_at DESC
            """
        ).fetchall()
        _tick("global_note")
        problematic_feeds = get_problematic_feeds_cached(conn)
        _tick("problematic_feeds")
        feed_urls = folder_feed_urls_by_id.get(selected_folder_id, set())

    meta_block_ms = int((time.perf_counter() - start_req) * 1000)
    LOGGER.info("[perf] home: meta_block=%dms", meta_block_ms)

    filtered_feed_urls = filter_feed_urls(feed_urls, list_feed_url)
    selected_feed_url = list_feed_url
    selected_tag = normalize_tag_value(tag)
    selected_feed_tag = normalize_tag_value(feed_tag)
    selected_query = normalize_search_query(q)
    legacy_saved_mode = (read_filter or "").strip().lower() == "saved"
    selected_read_filter = normalize_read_filter(read_filter)
    selected_star_only = normalize_star_only(star_only) or legacy_saved_mode
    if selected_read_filter == "capture_failed" and not selected_star_only:
        selected_read_filter = "all"  # a Saved-only filter must not leak into Feeds (e.g. via History's resume link)
    # Scope-tab landing state (`home=1`, either mode; legacy `saved_home=1` for
    # Saved): the tab shows its tree without loading any posts (All can be huge)
    # — the user picks All / a folder / a feed explicitly.
    selected_home = bool(home) or (bool(saved_home) and selected_star_only)
    selected_saved_home = selected_home and selected_star_only
    if selected_home:
        selected_feed_url = None
        selected_tag = None
        selected_feed_tag = None
        filtered_feed_urls = set()
    selected_resume_read_filter = normalize_resume_read_filter(resume_read_filter)
    # Respect an explicit resume_read_filter provided by the caller. Only
    # default to the current read selection when no explicit resume value was
    # supplied (so tag navigation can pass the previous view in resume_read_filter).
    if resume_read_filter is None and selected_read_filter in {"all", "unread"}:
        selected_resume_read_filter = selected_read_filter

    if selected_read_filter == "history":
        filtered_feed_urls = all_feed_urls
        selected_feed_url = None
        selected_tag = None
        selected_feed_tag = None
        selected_star_only = False

    tag_start = time.perf_counter()
    # The landing (home=1) suppresses POSTS, not sidebar navigation — the folder
    # tree still renders, so the sidebar Tags list should too. filtered_feed_urls
    # was emptied above for the post query, which would also blank the tag list;
    # scope tags to the folder's feeds instead. Without this, deleting a tag
    # (which reloads onto the landing) makes the whole Tags list look empty even
    # though the tags still exist.
    tag_scope_feed_urls = feed_urls if selected_home else filtered_feed_urls
    tag_rows = get_tag_counts_for_feeds(tag_scope_feed_urls)
    tag_block_ms = int((time.perf_counter() - tag_start) * 1000)
    LOGGER.info("[perf] home: tag_block=%dms", tag_block_ms)

    gap_start = time.perf_counter()
    try:
        _b1 = time.perf_counter()
        saved_unread_count = get_saved_unread_count()
        unread_count_ms = int((time.perf_counter() - _b1) * 1000)
        _b2 = time.perf_counter()
        saved_counts_by_folder = get_saved_counts_by_folder(folder_feed_urls_by_id)
        counts_by_folder_ms = int((time.perf_counter() - _b2) * 1000)
        _b3 = time.perf_counter()
        inbox_total = get_starred_inbox_total()
        inbox_total_ms = int((time.perf_counter() - _b3) * 1000)
    except Exception as exc:  # noqa: BLE001 — badges only; never block the render
        LOGGER.warning("saved counts failed: %s", exc)
        saved_unread_count = 0
        saved_counts_by_folder = {}
        inbox_total = 0
        unread_count_ms = counts_by_folder_ms = inbox_total_ms = -1
    badges_ms = int((time.perf_counter() - gap_start) * 1000)
    LOGGER.info(
        "[perf] home: badges_detail unread_count=%dms counts_by_folder=%dms inbox_total=%dms",
        unread_count_ms,
        counts_by_folder_ms,
        inbox_total_ms,
    )

    title_map_start = time.perf_counter()
    feed_title_map = get_feed_title_map()
    title_map_ms = int((time.perf_counter() - title_map_start) * 1000)
    inactive_feeds = [
        {
            "feed_url": str(r["feed_url"]),
            "feed_title": feed_title_map.get(str(r["feed_url"]), str(r["feed_url"])),
            "disabled_at": str(r["disabled_at"] or ""),
            "reason": str(r["reason"] or ""),
            "disabled_label": _disabled_label(str(r["reason"] or ""), str(r["disabled_at"] or "")),
            "folder_id": r["folder_id"],
            "folder_name": str(r["folder_name"] or ""),
        }
        for r in inactive_feed_rows
    ]
    # Sort alphabetically by title (the SQL orders by disabled_at, which reads as
    # random in the list); title isn't known until feed_title_map is applied above.
    inactive_feeds.sort(key=lambda x: x["feed_title"].casefold())
    problematic_unseen_count = 0
    _needs_replacement_urls = get_feeds_needing_replacement()
    for problematic_feed in problematic_feeds:
        pf_url = cast(str, problematic_feed["feed_url"])
        problematic_feed["feed_title"] = feed_title_map.get(pf_url, pf_url)
        problematic_feed["needs_replacement"] = pf_url in _needs_replacement_urls
        problematic_feed["can_suggest_migration"] = feed_discovery.is_known_dead_end_host(pf_url)
        pf_last_failure_at = problematic_feed.get("last_failure_at")
        if not isinstance(pf_last_failure_at, (int, float)):
            continue
        if problematic_feed.get("acknowledged_at"):
            continue
        if problematic_feeds_last_viewed_at is None or float(pf_last_failure_at) > problematic_feeds_last_viewed_at:
            problematic_unseen_count += 1
    error_feed_urls: set[str] = {cast(str, pf["feed_url"]) for pf in problematic_feeds if not pf.get("acknowledged_at")}
    feeds_by_folder: dict[int, list[FeedInFolder]] = {}
    # folder_id → does it hold any ACTIVE feed. The tree needs this for every
    # folder to decide whether to draw an expand toggle, but only needs the rows
    # themselves for the two folders it renders inline (see the loop below).
    folder_has_feeds: dict[int, bool] = {}
    # feed_url → containing_folder_id, so feed-name links in posts/entry can
    # navigate to the feed's own folder rather than the currently-viewed one.
    feed_to_folder: dict[str, int] = {}
    # Only two folders ever render their feed rows in this response: the selected
    # one (inline, so the active-feed highlight and auto-expand survive a full
    # page load) and the root. Every other folder ships an empty <ul> that fills
    # from /tree/folder-feeds/{id} on first expand, and needs nothing here but
    # "does it have any feeds" to decide whether to draw the expand toggle.
    #
    # Building all of them anyway cost a FeedInFolder per feed — 2,867 of them on
    # the live library — plus a urlparse/quote_plus for each icon URL and a sort
    # per folder, on every single page load. That is ~0.5s of pure Python in the
    # request path at rest, and several seconds when a refresh pass is running and
    # its worker threads are competing for the GIL. The lazy-rendering pass that
    # made the ROWS lazy left the BUILDING eager; this closes that gap.
    feeds_rendered_for = {selected_folder_id, root_id}
    for row in folder_rows:
        folder_row_id = cast(int, row["id"])
        urls = direct_feed_urls_by_folder.get(folder_row_id, [])
        # Needed for every folder regardless: the post list maps each post's feed
        # back to its folder through this.
        for url in urls:
            feed_to_folder[url] = folder_row_id
        # The tree hides disabled feeds, so "has feeds" means "has active feeds"
        # — otherwise a folder holding only disabled feeds would draw a toggle
        # that expands to nothing.
        folder_has_feeds[folder_row_id] = any(url not in disabled_feed_urls for url in urls)
        if folder_row_id not in feeds_rendered_for:
            continue
        all_folder_feeds = [
            FeedInFolder(
                url=url,
                title=feed_title_map.get(url, url),
                icon_url=get_favicon_url(url),
                unread_count=unread_counts_by_feed.get(url, 0),
                has_error=url in error_feed_urls,
                disabled=url in disabled_feed_urls,
            )
            for url in urls
        ]
        # Active feeds first (alphabetical), disabled greyed at the bottom.
        all_folder_feeds.sort(key=lambda f: (f.disabled, f.title.casefold()))
        active_folder_feeds = [f for f in all_folder_feeds if not f.disabled]
        _disambiguate_feed_titles(active_folder_feeds)
        feeds_by_folder[folder_row_id] = active_folder_feeds

    root_folder_row = next((row for row in folder_rows if cast(int, row["depth"]) == 0), None)
    child_folder_rows = [row for row in folder_rows if cast(int, row["depth"]) == 1]
    # The Settings → Feeds folders table and stale list are NOT built here:
    # they render a row per feed (megabytes at thousands of feeds), so
    # /settings/feeds/panel/{folders,stale} serves them on first open.

    # Determine server-side limit. If the client requested a "chunk" (page) use
    # CHUNK_SIZE per chunk; otherwise use the default limit from list_entries_for_feeds.
    #
    # A chunk_delta continuation's `chunk` value is an OFFSET -- the count of
    # real (post-dedup/filter) items the client already has, echoed back as
    # next_chunk below -- not a page multiplier. A plain `chunk` with no delta
    # (e.g. single-pane mode's initial small fetch) keeps the old page-count
    # meaning; that request has no "already have" state to offset from.
    offset = None
    # "Filter this view" pages through the view's matches: chunk is the position in the UNFILTERED list to scan from, filter_term
    # narrows it before slicing, and chunk_size is how many matches to return (not how many rows to scan).
    filtering = bool((filter_term or "").strip())
    delta_size = min(max(chunk_size, 1), 500) if chunk_size else CHUNK_SIZE
    try:
        if chunk and int(chunk) > 0:
            requested_chunk = int(chunk)
            if chunk_delta:
                offset = requested_chunk
                limit = 2000 if filtering else min(offset + delta_size, 2000)
            else:
                limit = min(requested_chunk * CHUNK_SIZE, 2000)
        else:
            limit = 250
    except Exception:
        limit = 250

    # Stamped before the list query: a post stored while it runs is newer than the stamp, so mark-read leaves it alone.
    view_loaded_at = time.time()
    posts_start = time.perf_counter()
    gap_ms = int((posts_start - gap_start) * 1000)
    LOGGER.info(
        "[perf] home: gap_block=%dms (badges=%dms title_map=%dms rest=%dms)",
        gap_ms,
        badges_ms,
        title_map_ms,
        gap_ms - badges_ms - title_map_ms,
    )
    # Exclude disabled feeds from the entry list unless the user has selected a
    # specific feed directly (clicking it should still let you browse its content).
    # Star mode keeps them: stars are deliberate curation and the saved-folder
    # badges count them, so hiding a disabled feed's saves made a folder show a
    # badge of 4 with an empty list.
    entry_feed_urls = filtered_feed_urls if (list_feed_url or selected_star_only) else filtered_feed_urls - disabled_feed_urls
    # The Kept view browses kept-but-unsubscribed feeds too: they're hidden from
    # the tree/All Feeds but their starred/tagged entries (with real tags) still
    # live in reader, so add them to the scan scope when in the saved/kept view.
    #
    # The Saved Articles feed belongs here for a different reason: it is a live
    # reader feed that sits in NO folder, so the folder-derived set above misses
    # it entirely. Its posts then reached the list only through the orphan-archive
    # merge — which carries the ARCHIVED title and no thumbnail, so an edited
    # title showed correctly in the article header (read from reader) and stale in
    # the list, and the thumbnail appeared on open and vanished on return.
    #
    # ONLY at the root or Uncategorized, and only when no feed is selected:
    # belonging to no folder means it belongs to the whole-library view and
    # the unfoldered subset — same as any other orphan feed — not to every
    # folder. Adding it unconditionally put saved articles in every folder's
    # list; only gating on root left it unreachable from Saved's own
    # Uncategorized grouping once Uncategorized's shared feed-set (used by
    # both modes) stopped including it by default (2026-08-27).
    if selected_star_only and selected_folder_id in (root_id, UNCATEGORIZED_FOLDER_ID) and not selected_feed_url:
        # kept_feeds belongs here for the same reason SAVED_FEED_URL does: a
        # kept-but-unsubscribed feed has no folder_feeds row any more, so it
        # belongs to the whole-library view, not every folder. Missing this
        # gate put a kept feed's posts in every folder's Saved list — reported
        # live 2026-09-17 for a feed that had been unsubscribed and later
        # resubscribed, leaving a stale kept_feeds row (harmless on its own;
        # the leak was in never scoping the union at all).
        entry_feed_urls = entry_feed_urls | get_kept_feed_urls()
        entry_feed_urls = entry_feed_urls | {saved_articles_service.SAVED_FEED_URL}
    posts = list_entries_for_feeds(
        entry_feed_urls,
        limit=limit,
        sort_by=selected_sort_by,
        sort_dir=selected_sort_dir,
        read_filter=selected_read_filter,
        star_only=selected_star_only,
        selected_tag=selected_tag,
        selected_feed_tag=selected_feed_tag,
        search_query=selected_query,
        # The Inbox is the to-do pile: stars only. Every other Saved node still
        # spans the whole kept set (starred OR tagged), because a tag is filing
        # and filing something is not a to-do.
        kept_scope=("starred" if inbox_view else "kept"),
        # The initial load's client reveals ~20 rows at a time and pages by offset, so a page a few rows short is fine.
        # A filtered scan wants the 2,000-row window as it is; the retry would double it (4,000 rows) to backfill hidden rows.
        min_fill=0.0 if filtering else (0.9 if offset is None else 1.0),
    )

    # Surface orphan archive entries (saved articles whose feed has been
    # unsubscribed). Two entry points, both gated on the saved filter. A search
    # query narrows either (metadata match, passed through as search_terms)
    # rather than excluding orphans outright:
    #   1. Root "All Feeds" with no feed/tag — orphans belong to no folder, so
    #      per-folder views legitimately exclude them.
    #   2. A specific feed selected that is no longer live (the user clicked
    #      the feed link on an orphaned save) — show just that feed's archive.
    orphan_only_feed = selected_feed_url if (selected_star_only and selected_feed_url and selected_feed_url not in all_feed_urls) else None
    if orphan_only_feed:
        try:
            posts = merge_orphan_saved_entries(
                posts,
                live_feed_urls=all_feed_urls,
                sort_by=selected_sort_by,
                sort_dir=selected_sort_dir,
                limit=limit,
                only_feed_url=orphan_only_feed,
                search_terms=search_terms_from_query(selected_query),
                kept_scope=("starred" if inbox_view else "kept"),
                capture_failed=selected_read_filter == "capture_failed",
            )
        except Exception as exc:  # noqa: BLE001
            LOGGER.warning("orphan saved entry merge (feed %s) failed: %s", orphan_only_feed, exc)
    elif (
        selected_star_only
        and not selected_saved_home
        and selected_folder_id == root_id
        and not selected_feed_url
        and not selected_tag
        # Orphans are read by definition (no live entry to be unread), so a
        # Saved view narrowed to unread excludes them.
        and selected_read_filter != "unread"
    ):
        try:
            posts = merge_orphan_saved_entries(
                posts,
                # The Saved Articles feed is LIVE in reader but sits in no
                # folder, so a folder-derived set makes every one of its posts
                # look orphaned. They then render from the archive: the archived
                # title (so an edited title showed stale in the list and correct
                # in the header) and no thumbnail (so it appeared on open and
                # vanished on return). An entry whose feed is live is not an
                # orphan, whatever the folder tree says.
                live_feed_urls=all_feed_urls | {saved_articles_service.SAVED_FEED_URL},
                search_terms=search_terms_from_query(selected_query),
                sort_by=selected_sort_by,
                sort_dir=selected_sort_dir,
                limit=limit,
                kept_scope=("starred" if inbox_view else "kept"),
                capture_failed=selected_read_filter == "capture_failed",
            )
        except Exception as exc:  # noqa: BLE001
            LOGGER.warning("orphan saved entry merge failed: %s", exc)

    posts_block_ms = int((time.perf_counter() - posts_start) * 1000)
    LOGGER.info("[perf] home: posts_block=%dms", posts_block_ms)

    # The offset the client should ask for next -- the TRUE count of real
    # (post-dedup/filter/orphan-merge) items now available, not a page-number
    # arithmetic guess. Must be captured here: after dedup/orphan-merge have
    # had their say, before the chunk_delta slice below narrows `posts` down
    # to just this request's own delta.
    #
    # The previous scheme derived "next chunk" from the REQUESTED limit
    # (`limit // CHUNK_SIZE + 1`), fixed 2026-09-05 to stop deriving it from
    # the CLIENT's rendered count (which could undercount from its own
    # display-only filtering and get permanently stuck re-fetching the same
    # slice). That fix didn't cover this: SERVER-side dedup (cross-feed
    # duplicates -- the same article saved once via a live feed and once via
    # a bookmarklet/extension capture, common in a large Saved/Kept backlog)
    # can *also* shrink `len(posts)` below the requested `limit`, and by a
    # DIFFERENT amount at each successively larger `limit` as more duplicate
    # pairs enter the window. Page-number-based slicing (`(chunk-1)*SIZE :
    # chunk*SIZE`) assumed that gap was always zero, so a later chunk's fixed
    # offset silently permanently skipped whatever an earlier chunk's dedup
    # pass had already removed -- visible live as "Biggest first" showing
    # entries out of size order a few chunks into scrolling a Saved view with
    # cross-posted duplicates. Offsetting from the actual `len(posts)` here
    # instead of a formula makes the slice correct regardless of how much
    # dedup happens at any point, since windowed sorts (_sorted_star_key_window)
    # are a stable, prefix-preserving function of the key set: a bigger `limit`
    # only ever appends past what a smaller one already returned, never
    # reorders it.
    next_chunk = len(posts)
    view_exhausted = False
    # Refresh-contention investigation (Plan.md Tier 1): a 2026-09-03 live capture
    # (folder_id=9, 12961ms total) accounted for only ~7s across meta/tag/gap/posts
    # blocks -- ~6s was unattributed, after posts_block and before the response
    # finished sending. ctx_block below times the context-dict build (settings/
    # integration-status lookups between here and the template call); the
    # StreamingResponse wrapper further down times the Jinja render/stream itself,
    # since .stream() is lazy and actually runs after this function returns.
    _ctx_start = time.perf_counter()

    # Prioritize lead-image backfill for entries currently visible in this chunk.
    # Fire-and-forget: returns immediately; semaphore prevents concurrent pile-up.
    uncached_posts = [p for p in posts if not p.get("thumbnail_url")]
    # Honours LECTIO_DISABLE_STARTUP_BACKFILL for the reason conftest sets it,
    # and this is the daemon that was behind the long-running "database is
    # locked" CI flake. Caught 2026-08-09 by the diagnostics dump: the failing
    # run's thread list held one `_run_in_user_context` thread inside
    # `backfill_entry_list → _fetch_source_lead_image → is_safe_outbound_url →
    # socket.getaddrinfo`, i.e. a DNS lookup, while the test's own fixture was
    # opening the same per-user meta DB. That switch already covered the
    # *startup* daemons and the per-request media scan; rendering a post list
    # spawns this one, so any test that renders a list could lose the race —
    # which is exactly why it kept failing in tests that had nothing to do with
    # the branch under review.
    if uncached_posts and os.getenv("LECTIO_DISABLE_STARTUP_BACKFILL", "0") == "1":
        uncached_posts = []
    if uncached_posts:
        # Re-bind the request's tenancy user inside the daemon thread; a bare
        # thread does not inherit contextvars, so backfill_entry_list would
        # otherwise run unbound and raise TenancyUnboundError.
        _bf_uid = tenancy.current_user_id()
        threading.Thread(
            target=_run_in_user_context,
            args=(_bf_uid, lead_image_service.backfill_entry_list, uncached_posts),
            daemon=True,
        ).start()

    # If the client requested a delta chunk (incremental load), return only
    # the slice for that chunk rather than the cumulative list up to limit.
    # `offset` (set above, when chunk_delta was given) is the true prior
    # count, not a page-number multiple -- see next_chunk's comment above.
    try:
        if chunk and chunk_delta and offset is not None:
            if filtering:
                from routes.entries import _view_filter_predicate  # lazy: routes.entries imports from main

                matches_filter = _view_filter_predicate(filter_term, folder_id=folder_id)
                scanned = posts[offset:]
                picked: list[dict] = []
                consumed = 0
                for post in scanned:
                    consumed += 1
                    if matches_filter(post):
                        picked.append(post)
                        if len(picked) >= delta_size:
                            break
                view_exhausted = consumed >= len(scanned)
                next_chunk = offset + consumed
                posts = picked
            else:
                posts = posts[offset : offset + delta_size]
    except Exception:
        # On any error, fall back to the cumulative behavior.
        pass

    selected_entry = None
    if feed_url and entry_id:
        selected_entry = get_entry_detail(feed_url, entry_id)
        if selected_entry and not selected_entry["read"]:
            with get_reader() as reader:
                reader.mark_entry_as_read((feed_url, entry_id))
            try:
                upsert_entry_read_state(feed_url, entry_id)
            except Exception:
                LOGGER.warning("upsert_entry_read_state failed in home (db contention?); entry still marked read in reader", exc_info=True)
            with unread_counts_cache_lock:
                _bump_unread_counts_generation()
                unread_counts_cache.clear()
            selected_entry["read"] = True
            for post in posts:
                if post["feed_url"] == feed_url and post["id"] == entry_id:
                    post["read"] = True
                    break

    # Context-aware browser/tab title: "Lectio — <context>" (plain "Lectio" at
    # root). Rendered into <title>; the client syncs document.title on pane-swap
    # so the browser Back list gets meaningful, distinct entries.
    _title_ctx: str | None = None
    if selected_entry and str(selected_entry.get("title") or "").strip():
        _title_ctx = str(selected_entry["title"]).strip()
    elif selected_query:
        _title_ctx = f"Search: {selected_query}"
    elif selected_tag:
        _title_ctx = f"#{selected_tag}"
    elif selected_feed_url:
        try:
            with get_reader() as _tr:
                _tf = _tr.get_feed(selected_feed_url, None)
            if _tf is not None:
                _title_ctx = str(getattr(_tf, "user_title", None) or _tf.title or selected_feed_url)
        except Exception:  # noqa: BLE001
            _title_ctx = None
    elif selected_star_only:
        _title_ctx = "Saved"
    elif selected_folder_id == UNCATEGORIZED_FOLDER_ID:
        _title_ctx = UNCATEGORIZED_FOLDER_NAME
    elif selected_folder_id and selected_folder_id != root_id:
        try:
            _title_ctx = next(
                (str(r["name"]) for r in folder_rows if cast(int, r["id"]) == int(selected_folder_id)),
                None,
            )
        except Exception:  # noqa: BLE001
            _title_ctx = None
    page_title = f"Lectio — {_title_ctx}" if _title_ctx else "Lectio"

    # Auto-refresh cadence shown in the menu: the bound user's own value in multi
    # mode (the menu form posts to set *their* setting), the cached global in single.
    _arm = _effective_auto_refresh_minutes()
    _tmpl_ctx = {
        "page_title": page_title,
        "portrait_img_max_width": get_portrait_img_max_width(),
        "mark_read_on_new_tab": mark_read_on_new_tab_enabled(),
        "request": request,
        "folder_rows": folder_rows,
        "root_folder_row": root_folder_row,
        "child_folder_rows": child_folder_rows,
        "folder_options": folder_options,
        "feeds_by_folder": feeds_by_folder,
        "folder_has_feeds": folder_has_feeds,
        "feed_to_folder": feed_to_folder,
        "unsubscribed_feed_urls": unsubscribed_feed_urls_among(
            [p.get("feed_url") for p in posts] + ([selected_entry.get("feed_url")] if selected_entry else [])
        ),
        "push_feed_urls": get_push_active_feed_urls(),
        "tag_rows": tag_rows,
        "all_tag_names": get_all_manual_tag_names(),
        "selected_folder_id": selected_folder_id,
        # Labels the phone's "up to the folder" control when the list is scoped
        # to one feed, so the button says where it goes instead of "Folders".
        "selected_folder_name": (
            _selected_folder_name_ := next(
                (str(row["name"]) for row in folder_rows if cast(int, row["id"]) == selected_folder_id),
                "",
            )
        ),
        # Gates duration-syntax parsing in "Filter this view" (app.js) to the
        # one folder it's meaningful in, without depending on the Settings
        # modal's lazily-fetched /settings/all data ever having loaded —
        # rendered directly so it's correct on first paint.
        "is_yt_folder": _selected_folder_name_ == get_yt_folder_name(),
        # The rules editor scopes an "Add to YT Playlist" rule's feed picker to
        # this folder (a playlist rule is never meaningful against a non-YT
        # feed) — rendered page-wide rather than looked up lazily so it works
        # the first time the rules panel opens, not just after visiting
        # Settings -> YouTube. None when the folder doesn't exist yet (never
        # auto-created just for this — that's YouTube Sync's job).
        "yt_folder_id": next(
            (cast(int, row["id"]) for row in folder_rows if str(row["name"]) == get_yt_folder_name()),
            None,
        ),
        "selected_feed_url": selected_feed_url,
        "selected_tag": selected_tag,
        "selected_feed_tag": selected_feed_tag,
        "selected_query": selected_query,
        "problematic_feeds": problematic_feeds,
        "problematic_feed_count": len(problematic_feeds),
        "problematic_unseen_count": problematic_unseen_count,
        "selected_sort_by": selected_sort_by,
        "selected_sort_dir": selected_sort_dir,
        "selected_read_filter": selected_read_filter,
        "has_capture_failed": bool(selected_star_only and starred_archive_service.has_capture_failed()),
        "selected_star_only": selected_star_only,
        "selected_saved_home": selected_saved_home,
        "selected_home": selected_home,
        "selected_resume_read_filter": selected_resume_read_filter,
        "saved_unread_count": saved_unread_count,
        "saved_counts_by_folder": saved_counts_by_folder,
        "inbox_view": inbox_view,
        "inbox_total": inbox_total,
        "global_note": global_note,
        "email_configured": is_email_configured(),
        "yt_oauth_connected": youtube_oauth_connected(),
        "yt_embed_account_features": youtube_embed_account_features_enabled(),
        "pinterest_oauth_connected": pinterest_oauth_connected(),
        "pinterest_connected": pinterest_oauth_connected(),
        "inoreader_connected": inoreader_connected(),
        "pinterest_configured": bool(_ENV_PINTEREST_OAUTH_CLIENT_ID and _ENV_PINTEREST_OAUTH_CLIENT_SECRET),
        "email_to_default": email_to_default,
        "instapaper_configured": is_instapaper_configured(),
        "quire_configured": is_quire_configured(),
        "quire_project_name": quire_project_name(),
        "reddit_connected": reddit_connected(),
        "youtube_sync_last_at": youtube_sync_last_at,
        "youtube_sync_last_result": youtube_sync_last_result,
        "inactive_feeds": inactive_feeds,
        "inactive_feed_count": len(inactive_feeds),
        "posts": posts,
        "next_chunk": next_chunk,
        "chunk_size": CHUNK_SIZE,
        # An empty *delta* chunk means "no more to load" (infinite-scroll end),
        # not "nothing matches" — the empty-state message must only render on
        # a full/initial fetch, or it would get appended below real posts.
        "is_chunk_delta_fetch": bool(chunk and chunk_delta),
        "view_exhausted": view_exhausted,
        "view_loaded_at": view_loaded_at,
        "selected_entry": selected_entry,
        "message": message,
        "no_rss_url": no_rss_url,
        "force_url": force_url,
        # Quick-subscription deep link (e.g. RSSHub-Radar pointed at Lectio via
        # the Feedbin/Nextcloud News override). Only pass through http(s) URLs so
        # the auto-opened Add Feed dialog can't be primed with a junk scheme.
        "subscribe_url": subscribe if (subscribe or "").strip().lower().startswith(("http://", "https://")) else None,
        "auto_refresh_enabled": _arm > 0,
        "auto_refresh_minutes": _arm,
        "auto_refresh_option_minutes": AUTO_REFRESH_OPTION_MINUTES,
        "static_asset_version": STATIC_ASSET_VERSION,
        "debug_mode": DEBUG_MODE,
        "highlight_rules": highlight_rules,
        "email_contacts": email_contacts,
        "email_bcc": email_bcc,
        "profile_name": profile_name,
        "profile_email": profile_email,
        "profile_avatar_url": profile_avatar_url,
        "current_user": _current_web_username(request),
        "is_admin": _is_web_admin(_current_web_user(request)),
        "current_api_token": (user_store.get_api_token(_uid) if (_uid := _current_web_user(request)) else ""),
        # Uncategorized-only feeds count (e.g. a fresh install whose first feed
        # is a bookmarklet-created Saved Articles feed) — the toolbar must render.
        "no_feeds": len(all_feed_urls) == 0 and len(all_reader_feed_urls) == 0,
    }
    ctx_block_ms = int((time.perf_counter() - _ctx_start) * 1000)
    if ctx_block_ms > 100:
        LOGGER.info("[perf] home: ctx_block=%dms", ctx_block_ms)
    _stream = templates.env.get_template("index.html").stream(_tmpl_ctx)
    _stream.enable_buffering(50)

    def _timed_stream():
        # See the ctx_block comment above: .stream() is lazy, so the actual
        # Jinja render (and its DB/Python work per post) happens as this
        # generator is drained by StreamingResponse, after _home_inner returns.
        _render_start = time.perf_counter()
        try:
            yield from _stream
        finally:
            render_ms = int((time.perf_counter() - _render_start) * 1000)
            if render_ms > 100:
                LOGGER.info("[perf] home: template_stream=%dms", render_ms)

    # Stateful, per-user HTML: never let a browser or CDN edge (Cloudflare
    # fronts at least one deployment) cache it per-URL — a stale cached page
    # ships stale inline JS and resurrects long-fixed bugs.
    return StreamingResponse(_timed_stream(), media_type="text/html", headers={"Cache-Control": "no-store"})


# Thumbnail dimensions: 2× the CSS slot (4.5rem ≈ 72px, tile height ≈ 84px) for retina.
_THUMB_W = 144
_THUMB_H = 168
# Bump when the RENDERING changes in a way that makes already-cached thumbnails
# wrong. It is part of the cache key, so old entries are simply never looked up
# again and each thumbnail re-renders the first time it is actually viewed —
# no mass delete, and no refetch storm across a 59k-row cache.
#   a1 — flatten transparency onto white instead of `.convert("RGB")`, which
#        kept the RGB under the alpha and turned transparent line art (xkcd /
#        what-if illustrations, logos, diagrams) into black rectangles.
_THUMB_RENDER_VERSION = "a1"

# Cover-mode crop: map crop value → (horizontal fraction, vertical fraction), 0=start 1=end.
_THUMB_COVER_POS: dict[str, tuple[float, float]] = {
    "cover": (0.5, 0.5),
    "cover-top-left": (0.0, 0.0),
    "cover-top": (0.5, 0.0),
    "cover-top-right": (1.0, 0.0),
    "cover-left": (0.0, 0.5),
    "cover-right": (1.0, 0.5),
    "cover-bottom-left": (0.0, 1.0),
    "cover-bottom": (0.5, 1.0),
    "cover-bottom-right": (1.0, 1.0),
    "left": (0.0, 0.5),  # legacy alias
}


STARRED_ASSET_URL_PREFIX = "/starred-asset/"


def _guid_type(ids: list[str]) -> str:
    """Return 'url', 'string', or 'none' based on a sample of entry IDs."""
    if not ids:
        return "none"
    url_count = sum(1 for g in ids if g.startswith("http://") or g.startswith("https://"))
    return "url" if url_count > len(ids) // 2 else "string"


def _compare_one_feed(url: str) -> dict:
    """Fetch and parse one feed URL, returning metadata for the Add Feed comparison picker."""
    from services import url_guard

    _headers = {"User-Agent": "Lectio/1.0 (feed comparison; +https://github.com/joshg253/Lectio)"}
    try:
        with url_guard.build_client(timeout=10.0, headers=_headers) as client:
            resp = url_guard.safe_get(client, url, headers=_headers)
    except url_guard.UnsafeURLError:
        return {"url": url, "error": "This URL points at a private/internal address and cannot be fetched."}
    except Exception as exc:
        return {"url": url, "error": str(exc).split("\n")[0][:120]}
    if not resp.is_success:
        return {"url": url, "error": f"HTTP {resp.status_code}"}

    ct = resp.headers.get("content-type", "").lower()

    # JSON Feed (feedparser doesn't handle it)
    if "json" in ct:
        try:
            import json as _j

            data = _j.loads(resp.text)
            items = data.get("items", [])
            image_count = sum(
                1
                for item in items
                if item.get("image")
                or item.get("banner_image")
                or any(a.get("mime_type", "").startswith("image") for a in item.get("attachments") or [])
            )
            full_text_count = sum(1 for item in items if len(item.get("content_html") or item.get("content_text") or "") > 200)
            date_field = "none"
            if items:
                if items[0].get("date_published"):
                    date_field = "published"
                elif items[0].get("date_modified"):
                    date_field = "modified_only"
            latest = None
            if items:
                raw = items[0].get("date_published") or items[0].get("date_modified")
                if raw:
                    try:
                        latest = datetime.fromisoformat(raw.replace("Z", "+00:00")).strftime("%b %-d, %Y %H:%M")
                    except Exception:
                        latest = raw[:10]
            sample_title = next((i["title"] for i in items if i.get("title")), None)
            if sample_title:
                sample_title = html.unescape(sample_title)
            return {
                "url": url,
                "format": "JSON Feed",
                "title": data.get("title"),
                "entry_count": len(items),
                "image_count": image_count,
                "full_text": full_text_count > len(items) // 2 if items else False,
                "date_field": date_field,
                "guid_type": _guid_type([i.get("id", "") for i in items if i.get("id")]),
                "latest_date": latest,
                "sample_title": sample_title,
            }
        except Exception as exc:
            return {"url": url, "error": f"JSON parse error: {exc}"}

    # RSS / Atom via feedparser
    parsed = feedparser.parse(resp.text)
    if parsed.bozo and not parsed.entries:
        return {"url": url, "error": "Could not parse feed"}

    entries = parsed.entries
    image_count = 0
    full_text_count = 0
    for entry in entries:
        has_enc = any(enc.get("type", "").startswith("image") for enc in entry.get("enclosures", []))
        has_media = bool(entry.get("media_content") or entry.get("media_thumbnail"))
        if has_enc or has_media:
            image_count += 1
        content_val = (entry.get("content") or [{}])[0].get("value") or ""
        if len(content_val) > 200:
            full_text_count += 1

    date_field = "none"
    if entries:
        if entries[0].get("published_parsed"):
            date_field = "published"
        elif entries[0].get("updated_parsed"):
            date_field = "modified_only"

    version_map = {
        "rss20": "RSS 2.0",
        "rss10": "RSS 1.0",
        "rss092": "RSS 0.92",
        "rss091n": "RSS 0.91",
        "atom10": "Atom 1.0",
        "atom03": "Atom 0.3",
    }
    ver = parsed.get("version", "")
    fmt = version_map.get(ver) or (ver.upper() if ver else None)
    if not fmt:
        # feedparser didn't identify the version; sniff the body
        snip = resp.text[:2000].lower()
        if "<rss" in snip:
            fmt = "RSS"
        elif "<feed" in snip:
            fmt = "Atom"
        elif "<rdf:rdf" in snip:
            fmt = "RDF"
        else:
            fmt = "Feed"

    latest = None
    if entries:
        ts = entries[0].get("published_parsed") or entries[0].get("updated_parsed")
        if ts:
            try:
                latest = datetime(*ts[:6]).strftime("%b %-d, %Y %H:%M")
            except Exception:
                pass

    # Some feed generators double-encode entities in titles (WordPress's
    # wptexturize output making "Ocean's Dream" -> "Ocean&#8217;s Dream", then
    # the feed's own XML/HTML escaping turning that & into &amp; -- so the
    # XML parser's one pass of entity decoding leaves the literal text
    # "&#8217;s Dream" visible). Reported 2026-08-10 in the Compare picker.
    # A second unescape pass fixes the common case and is a no-op on an
    # already-clean title.
    sample_title = next((e.get("title") for e in entries if e.get("title")), None)
    if sample_title:
        sample_title = html.unescape(sample_title)

    return {
        "url": url,
        "format": fmt,
        "title": parsed.feed.get("title"),
        "entry_count": len(entries),
        "image_count": image_count,
        "full_text": full_text_count > len(entries) // 2 if entries else False,
        "date_field": date_field,
        "guid_type": _guid_type([e.get("id", "") for e in entries if e.get("id")]),
        "latest_date": latest,
        "sample_title": sample_title,
    }


def _devto_config_from_form(tag: str, top_days: str, english_only: str, min_reactions: str, tags_exclude: str) -> dict:
    """Shared form→config parsing for the add-feed and config-update routes.

    Checkbox semantics: the dialogs always post english_only as "1"/"0"; an
    omitted/empty field defaults to English-only in both routes.
    """
    return {
        "tag": tag.strip(),
        "top_days": int(top_days) if top_days.strip().isdigit() else None,
        "english_only": (english_only.strip() or "1").lower() not in ("0", "false", "off"),
        "min_reactions": int(min_reactions) if min_reactions.strip().isdigit() else None,
        "tags_exclude": tags_exclude.strip(),
    }


_SUBTITLE_SEPARATORS = (" - ", " – ", " — ", ": ", " | ")


def _site_name_from_subtitle(subtitle: str) -> str | None:
    """A feed's own <subtitle> is a human-authored site description, so it
    beats guessing from the URL when present -- reported 2026-08-10: TOSEC's
    feed carries subtitle "TOSEC - The Old School Emulation Center" while its
    domain (tosecdev.org) guesses nothing useful.

    Most subtitles pair a short name with a tagline via a separator ("Name -
    Tagline", "Name: Tagline") -- take just the name half. A subtitle with no
    separator is used whole if short enough to plausibly be a name rather
    than a full sentence; longer ones are too likely to be a description, so
    the caller falls back to the URL-based guess instead.
    """
    subtitle = subtitle.strip()
    if not subtitle:
        return None
    for sep in _SUBTITLE_SEPARATORS:
        if sep in subtitle:
            head = subtitle.split(sep, 1)[0].strip()
            if head:
                return head
    return subtitle if len(subtitle) <= 40 else None


def _auto_tag_webcomic_feeds() -> None:
    """Set strategy='webcomic' for all feeds in folders whose name contains 'comic'.

    Only sets non-manually-locked feeds so user overrides are respected.
    Runs at startup so new feeds added to comics folders are tagged immediately.
    """
    try:
        with get_meta_connection() as conn:
            rows = conn.execute(
                """
                SELECT DISTINCT ff.feed_url
                FROM folder_feeds ff
                JOIN folders f ON ff.folder_id = f.id
                WHERE lower(f.name) LIKE '%comic%'
                """
            ).fetchall()
            feed_urls = [str(r["feed_url"]) for r in rows]
            if not feed_urls:
                return
            now = time.time()
            for feed_url in feed_urls:
                existing = conn.execute(
                    "SELECT strategy, manual FROM feed_lead_image_strategy WHERE feed_url = ?",
                    (feed_url,),
                ).fetchone()
                if existing and existing["manual"]:
                    continue
                if existing and existing["strategy"] in ("webcomic", "artwork"):
                    continue
                conn.execute(
                    """
                    INSERT INTO feed_lead_image_strategy (feed_url, strategy, detected_at, manual)
                    VALUES (?, 'webcomic', ?, 0)
                    ON CONFLICT(feed_url) DO UPDATE SET
                        strategy = 'webcomic',
                        detected_at = excluded.detected_at
                    WHERE manual = 0
                    """,
                    (feed_url, now),
                )
                lead_image_service.store_feed_strategy(feed_url, "webcomic", manual=False)
    except Exception:
        LOGGER.exception("_auto_tag_webcomic_feeds failed")


def _auto_tag_github_release_feeds() -> None:
    """Set strategy='og_scrape' and suppress list thumbnails for GitHub release feeds.

    GitHub generates a unique social-preview card (og:image) per release, making
    og_scrape the right strategy.  Thumbnails are suppressed because the card is
    contextual, not a post image.  Skips feeds where strategy is manually locked.
    """
    try:
        with get_meta_connection() as conn:
            now = time.time()
            rows = conn.execute(
                "SELECT DISTINCT feed_url FROM folder_feeds WHERE lower(feed_url) LIKE '%github.com%/releases.atom'"
            ).fetchall()
            for row in rows:
                feed_url = str(row["feed_url"])
                existing = conn.execute(
                    "SELECT strategy, manual FROM feed_lead_image_strategy WHERE feed_url = ?",
                    (feed_url,),
                ).fetchone()
                if existing and existing["manual"]:
                    continue
                if not (existing and existing["strategy"] == "og_scrape"):
                    conn.execute(
                        """
                        INSERT INTO feed_lead_image_strategy (feed_url, strategy, detected_at, manual)
                        VALUES (?, 'og_scrape', ?, 0)
                        ON CONFLICT(feed_url) DO UPDATE SET
                            strategy = 'og_scrape',
                            detected_at = excluded.detected_at
                        WHERE manual = 0
                        """,
                        (feed_url, now),
                    )
                    lead_image_service.store_feed_strategy(feed_url, "og_scrape", manual=False)
                # Ensure row exists and thumbnail is off — unless the user pinned one. This runs on a
                # schedule, so without the guard it re-disabled thumbnails every pass and a deliberately
                # chosen icon kept vanishing hours after it was set, looking like the setting never saved.
                conn.execute(
                    "INSERT INTO feed_display_prefs (feed_url, show_lead_image_as_thumb)"
                    " VALUES (?, 0)"
                    " ON CONFLICT(feed_url) DO UPDATE SET show_lead_image_as_thumb = 0"
                    " WHERE feed_display_prefs.feed_thumbnail_url IS NULL",
                    (feed_url,),
                )
    except Exception:
        LOGGER.exception("_auto_tag_github_release_feeds failed")


# Domains whose feeds use the artwork image layout (image appended after text).
# Feed URLs matching any of these get strategy='artwork' automatically.
_ARTWORK_FEED_DOMAINS = (
    "artstation.com",
    "deviantart.com",
)


def _auto_tag_artwork_feeds() -> None:
    """Set strategy='artwork' for feeds from known art-portfolio domains.

    Matches on feed URL, not folder name.  Skips manually-locked strategies.
    Runs at startup so new and existing feeds are migrated automatically.
    """
    try:
        with get_meta_connection() as conn:
            now = time.time()
            for domain in _ARTWORK_FEED_DOMAINS:
                rows = conn.execute(
                    "SELECT DISTINCT feed_url FROM folder_feeds WHERE lower(feed_url) LIKE ?",
                    (f"%{domain}%",),
                ).fetchall()
                for row in rows:
                    feed_url = str(row["feed_url"])
                    existing = conn.execute(
                        "SELECT strategy, manual FROM feed_lead_image_strategy WHERE feed_url = ?",
                        (feed_url,),
                    ).fetchone()
                    if existing and existing["manual"]:
                        continue
                    if existing and existing["strategy"] == "artwork":
                        continue
                    conn.execute(
                        """
                        INSERT INTO feed_lead_image_strategy (feed_url, strategy, detected_at, manual)
                        VALUES (?, 'artwork', ?, 0)
                        ON CONFLICT(feed_url) DO UPDATE SET
                            strategy = 'artwork',
                            detected_at = excluded.detected_at
                        WHERE manual = 0
                        """,
                        (feed_url, now),
                    )
                    lead_image_service.store_feed_strategy(feed_url, "artwork", manual=False)
    except Exception:
        LOGGER.exception("_auto_tag_artwork_feeds failed")


_FEED_THUMB_CACHE_PREFIX = "feedthumb:"
_FEED_THUMB_MAX_BYTES = 4 * 1024 * 1024


def _feed_thumb_cache_key(feed_url: str) -> str:
    """Keyed by feed, not by URL, so re-pinning replaces the copy and an expiring source URL cannot
    orphan it. Hashed because the key is a primary key and feed URLs are long and arbitrary."""
    return _FEED_THUMB_CACHE_PREFIX + hashlib.sha1(feed_url.encode("utf-8", "replace")).hexdigest()


def _pin_feed_thumbnail_bytes(feed_url: str, thumbnail_url: str) -> bool:
    """Copy a pinned thumbnail into the image cache so it survives its source URL.

    Signed-CDN thumbnails expire: a Telegram cdn4.telesco.pe link 404s within days, so a manually chosen
    icon kept vanishing and had to be re-set. `_IMG_CACHE_VOLATILE_PARAMS` cannot help — Telegram signs in
    the path, not the query string. Storing the bytes once is what makes "keep it until I change it" true.
    """
    try:
        with httpx.Client(follow_redirects=False, timeout=15.0) as client:
            resp = url_guard.safe_get(client, thumbnail_url, headers={"User-Agent": LECTIO_HONEST_USER_AGENT})
        resp.raise_for_status()
        body = resp.content
        content_type = (resp.headers.get("content-type") or "").split(";")[0].strip().lower()
        if not body or not content_type.startswith("image/") or len(body) > _FEED_THUMB_MAX_BYTES:
            return False
        _img_cache_store(_feed_thumb_cache_key(feed_url), body, content_type)
        return True
    except Exception:  # noqa: BLE001 — the URL still gets stored; this is only the durable copy
        LOGGER.info("[feed-thumb] could not pin bytes for %s", feed_url, exc_info=True)
        return False


def _drop_pinned_feed_thumbnail(feed_url: str) -> None:
    try:
        with get_img_cache_connection() as conn:
            conn.execute("DELETE FROM img_cache WHERE cache_key = ?", (_feed_thumb_cache_key(feed_url),))
    except Exception:
        LOGGER.debug("[feed-thumb] could not drop pinned bytes for %s", feed_url, exc_info=True)


def pinned_feed_thumbnail_keys() -> set[str]:
    """Every pinned-thumbnail cache key, for callers rendering a list of feeds.

    One query instead of one per row: the set is tiny (a pinned thumbnail is a deliberate per-feed act),
    while the list path asks about it once per entry.
    """
    try:
        with get_img_cache_connection() as conn:
            rows = conn.execute("SELECT cache_key FROM img_cache WHERE cache_key LIKE ?", (_FEED_THUMB_CACHE_PREFIX + "%",)).fetchall()
        return {str(r["cache_key"]) for r in rows}
    except Exception:
        return set()


def _pinned_thumb_response(feed_url: str) -> Response:
    """The one place a pinned thumbnail becomes a response, so /api/feed-thumb and /thumb cannot drift
    apart on cache headers or on what a miss looks like."""
    hit = _img_cache_get(_feed_thumb_cache_key(feed_url))
    if hit is None:
        return Response(status_code=404)
    body, content_type = hit
    return Response(
        content=body,
        media_type=content_type or "image/jpeg",
        headers={"Cache-Control": "private, max-age=86400"},
    )


def has_pinned_feed_thumbnail(feed_url: str) -> bool:
    try:
        with get_img_cache_connection() as conn:
            row = conn.execute("SELECT 1 FROM img_cache WHERE cache_key = ?", (_feed_thumb_cache_key(feed_url),)).fetchone()
        return row is not None
    except Exception:
        return False


@app.get("/api/feed-thumb")
def api_feed_thumb(feed_url: str = Query(...)):
    """Serve the pinned copy of a feed's chosen thumbnail. Stable URL, so the page never points at a
    signed CDN link that will expire."""
    return _pinned_thumb_response(feed_url)


# ---------------------------------------------------------------------------
# Per-entry lead-image pinning — the signed-URL-rot fix.
#
# 120K+ stored lead-image URLs, 22,903 of them DeviantArt wixmp links signed
# with a short-lived `?token=` JWT; only 2.5% ever had bytes land in the
# general /api/img cache, because that cache is populated by someone actually
# opening the article — most list thumbnails never trigger it. The article
# view has its own fix (_resign_expired_deviantart_url); this is the list-
# thumbnail one. Same mechanism as _feed_thumb_cache_key above, keyed per
# entry instead of per feed: pin small (thumbnail-sized, not full-res) bytes
# the moment a *signed* lead image is discovered, while the token is still
# fresh, so the list thumbnail never depends on that token again. See
# docs/architecture/images.md.
# ---------------------------------------------------------------------------

_ENTRY_THUMB_CACHE_PREFIX = "entrythumb:"
_ENTRY_THUMB_MAX_DIM = 400
_ENTRY_THUMB_TARGET_BYTES = 30_000
# A pinned thumbnail that is still bigger than this after downscaling (an animated GIF, or a page too large to decode) is not
# stored: /api/entry-thumb serves the bytes as-is, so 64 such rows (1.35 GB, up to 77 MB each) were being sent to the browser
# for a 400px list thumbnail, starving the page itself. Unpinned entries fall back to /thumb, which renders a static thumbnail.
_ENTRY_THUMB_MAX_STORE_BYTES = 1_000_000


def _entry_thumb_cache_key(feed_url: str, entry_id: str) -> str:
    """Keyed by (feed_url, entry_id), not by URL, for the same reason as
    _feed_thumb_cache_key: re-pinning replaces the copy in place and an
    expiring source URL cannot orphan it. sha256, not sha1 — CodeQL's
    py/weak-sensitive-data-hashing flags sha1 whenever a hashed value
    includes something matching an "id" pattern, even here where entry_id is
    a public feed-entry identifier, not a credential; this is just a cache
    key, so sha256 costs nothing and ends the false-positive noise."""
    digest = hashlib.sha256(f"{feed_url}\x1f{entry_id}".encode("utf-8", "replace")).hexdigest()
    return _ENTRY_THUMB_CACHE_PREFIX + digest


def _url_is_signed(url: str) -> bool:
    """True if the URL carries a query param this app already treats as a
    volatile signing token (see _IMG_CACHE_VOLATILE_PARAMS) — host-agnostic,
    so it flags any signing CDN, not just DeviantArt's wixmp."""
    try:
        params = parse_qsl(urlparse(url).query, keep_blank_values=True)
    except ValueError:
        return False
    return any(k.lower() in _IMG_CACHE_VOLATILE_PARAMS for k, _ in params)


def has_pinned_entry_thumbnail(feed_url: str, entry_id: str, *, usable_only: bool = False) -> bool:
    """usable_only ignores a pinned copy over _ENTRY_THUMB_MAX_STORE_BYTES (rows pinned before that cap existed): the list must
    not point at one, but the pin sink still counts it as pinned so it does not re-download the source on every pass."""
    try:
        with get_img_cache_connection() as conn:
            row = conn.execute(
                "SELECT 1 FROM img_cache WHERE cache_key = ?" + (" AND size <= ?" if usable_only else ""),
                (_entry_thumb_cache_key(feed_url, entry_id), *((_ENTRY_THUMB_MAX_STORE_BYTES,) if usable_only else ())),
            ).fetchone()
        return row is not None
    except Exception:
        return False


def _pin_entry_thumbnail_bytes(feed_url: str, entry_id: str, image_url: str) -> bool:
    """Fetch a signed lead-image URL once, while the token is fresh, and store a
    small (thumbnail-sized, not full-res) copy under a stable per-entry key that
    doesn't depend on the URL at all. Mirrors _pin_feed_thumbnail_bytes."""
    try:
        with httpx.Client(follow_redirects=False, timeout=15.0) as client:
            resp = url_guard.safe_get(client, image_url, headers={"User-Agent": LECTIO_HONEST_USER_AGENT})
        resp.raise_for_status()
        body = resp.content
        content_type = (resp.headers.get("content-type") or "").split(";")[0].strip().lower()
        if not body or not content_type.startswith("image/"):
            return False
        downscaled, new_ct = _maybe_downscale_image(body, _ENTRY_THUMB_MAX_DIM)
        if new_ct is not None:
            body, content_type = downscaled, new_ct
        body, content_type = _maybe_shrink_oversized_image(body, content_type, _ENTRY_THUMB_TARGET_BYTES)
        if len(body) > _ENTRY_THUMB_MAX_STORE_BYTES:
            LOGGER.info("[entry-thumb] not pinning %s/%s: %d bytes after downscaling", feed_url, entry_id, len(body))
            return False
        _img_cache_store(_entry_thumb_cache_key(feed_url, entry_id), body, content_type)
        return True
    except Exception:  # noqa: BLE001 — the raw URL still gets stored/shown; this is only the durable copy
        LOGGER.info("[entry-thumb] could not pin bytes for %s/%s", feed_url, entry_id, exc_info=True)
        return False


def _pin_entry_thumbnail_if_signed(feed_url: str, entry_id: str, image_url: str) -> None:
    """Sink wired into lead_image_service.store_entry_lead_image. Cheap no-op
    once pinned (a plain PK existence check), so calling it on every enhance
    pass for an already-pinned entry costs one indexed SQLite read."""
    if not _url_is_signed(image_url) or has_pinned_entry_thumbnail(feed_url, entry_id):
        return
    _pin_entry_thumbnail_bytes(feed_url, entry_id, image_url)


lead_image_service.set_thumb_pin_sink(_pin_entry_thumbnail_if_signed)


def _pinned_entry_thumb_response(feed_url: str, entry_id: str) -> Response:
    """The one place a pinned entry thumbnail becomes a response — mirrors
    _pinned_thumb_response so /api/entry-thumb and /thumb cannot drift apart."""
    hit = _img_cache_get(_entry_thumb_cache_key(feed_url, entry_id))
    if hit is None:
        return Response(status_code=404)
    body, content_type = hit
    return Response(
        content=body,
        media_type=content_type or "image/jpeg",
        headers={"Cache-Control": "private, max-age=86400"},
    )


@app.get("/api/entry-thumb")
def api_entry_thumb(feed_url: str = Query(...), entry_id: str = Query(...)):
    """Serve the pinned copy of an entry's lead-image thumbnail. Stable URL, so
    the list never points at a signed CDN link that will expire."""
    return _pinned_entry_thumb_response(feed_url, entry_id)


def _keep_existing_sensitive(key: str, str_val: str, sensitive: set[str]) -> bool:
    """True if a sensitive-field save should be ignored (leave the stored value).

    Masked secret fields reload blank in the UI, so saving the form with the
    field untouched sends "" (or the masked "••" placeholder). Treat both as
    "leave unchanged" so a routine re-save never silently wipes a stored secret.
    """
    return key in sensitive and (not str_val or str_val.startswith("••"))


_PRUNE_DELETE_BATCH = 500


def _parse_stored_dt(value) -> datetime | None:
    """Parse reader's naive-UTC 'YYYY-MM-DD HH:MM:SS[.ffffff]' or an ISO-'T'
    string from the meta DB. Returns None when absent/unparseable."""
    if not value:
        return None
    try:
        return datetime.fromisoformat(str(value))
    except ValueError:
        return None


def _prune_entries(
    feed_urls: list[str],
    *,
    read_cutoff: datetime | None = None,
    published_cutoff: datetime | None = None,
    include_unread: bool = False,
    dry_run: bool = False,
) -> int:
    """Bulk-delete old posts from *feed_urls* (retention + Purge utility core).

    Selection: with read_cutoff, read entries whose read time (meta read_at,
    falling back to reader's read_modified) is older; with published_cutoff,
    entries whose effective date (published/updated/added) is older — read
    only unless include_unread. Always protected: **any entry carrying a keep
    signal** — starred (TODO), manually tagged (filed), or archived (done) — plus
    the Saved Articles feed itself, plus a YouTube video that hasn't premiered
    yet (it hasn't delivered its content, so its age shouldn't count against
    it — a delayed premiere must not get swept before it ever airs). Archived
    is the newest of the four and the one worth stating: Archive removes the
    star, so without an explicit exemption an archived item would fall
    straight through into the prune, which is the opposite of "Archive keeps
    its contents".

    Deletes are tombstoned (deleted_entries) so a refresh can't resurrect
    copies still inside the publisher's feed window. Returns the count
    (deleted, or matching when dry_run)."""
    feed_urls = [u for u in feed_urls if not saved_articles_service.is_saved_articles_feed(u)]
    if not feed_urls or (read_cutoff is None and published_cutoff is None):
        return 0

    placeholders = ",".join("?" * len(feed_urls))
    with get_meta_connection() as conn:
        protected = {
            (str(r[0]), str(r[1]))
            for r in conn.execute(
                f"SELECT feed_url, entry_id FROM saved_entries WHERE feed_url IN ({placeholders})",
                feed_urls,
            )
        }
        protected |= {
            (str(r[0]), str(r[1]))
            for r in conn.execute(
                f"SELECT feed_url, entry_id FROM archived_entries WHERE feed_url IN ({placeholders})",
                feed_urls,
            )
        }
        read_at_map = {
            (str(r[0]), str(r[1])): _parse_stored_dt(r[2])
            for r in conn.execute(
                f"SELECT feed_url, entry_id, read_at FROM entry_read_state WHERE feed_url IN ({placeholders})",
                feed_urls,
            )
        }

    to_delete: list[tuple[str, str]] = []
    with get_reader() as reader:
        rows = (
            reader._storage.get_db()
            .execute(
                # Manually tagged entries are user-curated — never prune them.
                f"""SELECT e.feed, e.id, e.link, e.read, e.read_modified, e.published, e.updated, e.first_updated
                FROM entries e
                WHERE e.feed IN ({placeholders})
                  AND NOT EXISTS (SELECT 1 FROM entry_tags t
                                  WHERE t.feed = e.feed AND t.id = e.id
                                    AND t.key LIKE '{MANUAL_TAG_KEY_PREFIX}%')""",
                feed_urls,
            )
            .fetchall()
        )

        for feed, eid, link, read, read_modified, published, updated, first_updated in rows:
            pair = (str(feed), str(eid))
            if pair in protected:
                continue
            if _youtube_unpremiered_video_id(str(feed), link) is not None:
                continue
            if read_cutoff is not None:
                if not read:
                    continue
                read_time = read_at_map.get(pair) or _parse_stored_dt(read_modified)
                if read_time is None or read_time >= read_cutoff:
                    continue
            if published_cutoff is not None:
                if not read and not include_unread:
                    continue
                effective = _parse_stored_dt(published) or _parse_stored_dt(updated) or _parse_stored_dt(first_updated)
                if effective is None or effective >= published_cutoff:
                    continue
            to_delete.append(pair)

        if dry_run or not to_delete:
            return len(to_delete)

        for i in range(0, len(to_delete), _PRUNE_DELETE_BATCH):
            batch = to_delete[i : i + _PRUNE_DELETE_BATCH]
            with get_meta_connection() as conn:
                conn.executemany(
                    "INSERT OR REPLACE INTO deleted_entries (feed_url, entry_id, created_at) VALUES (?, ?, ?)",
                    [(fu, eid, datetime.now().isoformat()) for fu, eid in batch],
                )
                conn.executemany("DELETE FROM entry_read_state WHERE feed_url = ? AND entry_id = ?", batch)
            reader._storage.delete_entries(batch)

    invalidate_unread_counts_cache()
    return len(to_delete)


def _hard_delete_entry(reader, feed_url: str, entry_id: str, entry) -> None:
    """Tombstone and hard-delete one entry (shared by /entries/delete and
    /saved/deduplicate).

    reader.delete_entry only covers user-added entries; feed-provided ones go
    through the storage-level delete (the same API reader's own entry_dedupe
    plugin uses). The tombstone keeps the next refresh from resurrecting the
    entry while it's still in the publisher's feed window — the refresh service
    purges tombstoned entries after each update.
    """
    with get_meta_connection() as conn:
        conn.execute(
            "INSERT OR REPLACE INTO deleted_entries (feed_url, entry_id, created_at) VALUES (?, ?, ?)",
            (feed_url, entry_id, datetime.now().isoformat()),
        )
        conn.execute(
            "DELETE FROM saved_entries WHERE feed_url = ? AND entry_id = ?",
            (feed_url, entry_id),
        )
    if getattr(entry, "added_by", "") == "user":
        reader.delete_entry((feed_url, entry_id), missing_ok=True)
    else:
        reader._storage.delete_entries([(feed_url, entry_id)])


def deviantart_watch_feed_url() -> str | None:
    """file:// URL of the combined DeviantArt Watch feed, or None if it isn't set up."""
    with get_meta_connection() as conn:
        row = conn.execute("SELECT id FROM deviantart_feeds WHERE source = 'watch' LIMIT 1").fetchone()
    return deviantart_service.feed_file_url(str(row["id"])) if row else None


def apply_deviantart_unwatch_posts_action(username: str, action: str) -> int:
    """After un-Watching an artist, handle their posts already in the Watch feed.

    ``action``: ``keep`` (leave as-is), ``read`` (mark unread ones read) or ``purge``
    (tombstone + delete; starred, archived and tagged posts are always spared). Posts are
    matched by link (``deviantart.com/<user>/...``), not byline — most older posts have no
    author. Returns how many posts were marked read / deleted."""
    feed_url = deviantart_watch_feed_url()
    target = username.strip().lower()
    if action not in ("read", "purge") or not feed_url or not target:
        return 0
    like = "%deviantart.com/" + target.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "/%"
    with get_reader() as reader:
        rows = (
            reader._storage.get_db()
            .execute(
                f"""SELECT e.id, e.link, e.read FROM entries e
                WHERE e.feed = ? AND e.link LIKE ? ESCAPE '\\'
                  AND NOT EXISTS (SELECT 1 FROM entry_tags t WHERE t.feed = e.feed AND t.id = e.id
                                  AND t.key LIKE '{MANUAL_TAG_KEY_PREFIX}%')""",
                (feed_url, like),
            )
            .fetchall()
        )
        ids = [str(eid) for eid, link, read in rows if (deviantart_service.username_from_url(str(link or "")) or "").lower() == target]
        if action == "read":
            unread = {str(eid) for eid, _link, read in rows if not read}
            pairs = [(feed_url, eid) for eid in ids if eid in unread]
            for pair in pairs:
                reader.mark_entry_as_read(pair)
            if pairs:
                now_str = datetime.now().isoformat()
                with get_meta_connection() as conn:
                    conn.executemany(
                        "INSERT INTO entry_read_state (feed_url, entry_id, read_at) VALUES (?,?,?)"
                        " ON CONFLICT(feed_url, entry_id) DO UPDATE SET read_at=excluded.read_at",
                        [(fu, eid, now_str) for fu, eid in pairs],
                    )
                _bump_unread_counts_generation()
            invalidate_unread_counts_cache()
            return len(pairs)

        with get_meta_connection() as conn:
            protected = {
                str(r[0])
                for r in conn.execute(
                    "SELECT entry_id FROM saved_entries WHERE feed_url = ? UNION SELECT entry_id FROM archived_entries WHERE feed_url = ?",
                    (feed_url, feed_url),
                )
            }
        pairs = [(feed_url, eid) for eid in ids if eid not in protected]
        for i in range(0, len(pairs), _PRUNE_DELETE_BATCH):
            batch = pairs[i : i + _PRUNE_DELETE_BATCH]
            with get_meta_connection() as conn:
                conn.executemany(
                    "INSERT OR REPLACE INTO deleted_entries (feed_url, entry_id, created_at) VALUES (?, ?, ?)",
                    [(fu, eid, datetime.now().isoformat()) for fu, eid in batch],
                )
                conn.executemany("DELETE FROM entry_read_state WHERE feed_url = ? AND entry_id = ?", batch)
            reader._storage.delete_entries(batch)
    invalidate_unread_counts_cache()
    return len(pairs)


# Tested directly as main._ENTRY_LINK_MAX_LEN by tests/integration/test_entry_link_override.py;
# stays here (imported back into routes.entries) rather than moving with set_entry_link_route.
_ENTRY_LINK_MAX_LEN = 2000


# User-facing wording for a refused cleanup, keyed by ContentEditError.code.
# Authored here rather than taken from the exception so the response can never
# carry exception-derived text (CodeQL: py/stack-trace-exposure).
_CLEANUP_ERROR_MESSAGES = {
    "not_json": "The cleanup instructions were not valid JSON.",
    "empty": "No cleanup operations were sent.",
    "too_many": f"Too many operations in one go (max {content_edits.MAX_OPS}).",
    "bad_op": "One of the cleanup steps was malformed — reload the article and try again.",
    "no_html": "This entry has no HTML body to clean.",
    "unparseable": "This entry's HTML could not be parsed.",
    "would_empty": "That would remove the entire article body.",
}
_CLEANUP_ERROR_FALLBACK = "That cleanup could not be applied."


def _swap_host_in_url(url: str, host_map: dict[str, str]) -> str:
    """Swap *url*'s bare host per ``{from_host: to_host}`` (case-insensitive,
    ignores userinfo/port); scheme, path, query and fragment are preserved.
    Shared by the Edit-Website route and the apply_feed_url_rewrites script."""
    try:
        p = urlsplit(url)
    except ValueError:
        return url
    bare = (p.netloc or "").split("@")[-1].split(":")[0].lower()
    to = host_map.get(bare)
    return urlunsplit((p.scheme, to, p.path, p.query, p.fragment)) if to else url


def migrate_entry_to_new_host(reader, conn, feed, old_id, new_id, new_link) -> str:
    """Recreate one entry under its host-rewritten id, carrying the star, the
    archived (done) row, manual tags, read state and offline archive, then delete the
    old one. Returns "migrated" / "gone". Idempotent: an already-migrated entry
    no longer matches a from_host. Must run with the ingest rewrite hook in place
    so the feed re-serving the old guid updates the migrated entry, not a
    resurrected old one."""
    src = reader.get_entry((feed, old_id), None)
    if src is None:
        return "gone"
    if reader.get_entry((feed, new_id), None) is None:
        ed: dict = {"feed_url": feed, "id": new_id, "link": new_link or new_id, "title": src.title or ""}
        # entry_effective_date, not raw src.published — see the identical fix
        # (and its rationale) in _move_entry_to_feed's own synth path.
        _src_effective_date = entry_effective_date(src)
        if _src_effective_date:
            ed["published"] = _src_effective_date
        if getattr(src, "content", None):
            ed["content"] = [{"value": entry_content.entry_body(src)}]
        elif src.summary:
            ed["summary"] = src.summary
        reader.add_entry(ed)
    for t in reader.get_tags(src.resource_id):
        key = t[0] if isinstance(t, tuple) else t
        if key and key.startswith(MANUAL_TAG_KEY_PREFIX):
            reader.set_tag((feed, new_id), key)
            reader.delete_tag(src.resource_id, key, missing_ok=True)
    row = conn.execute(
        "SELECT saved_at FROM saved_entries WHERE feed_url = ? AND entry_id = ?",
        (feed, old_id),
    ).fetchone()
    if row:
        conn.execute(
            "INSERT OR IGNORE INTO saved_entries (feed_url, entry_id, saved_at) VALUES (?, ?, ?)",
            (feed, new_id, row["saved_at"]),
        )
        conn.execute("DELETE FROM saved_entries WHERE feed_url = ? AND entry_id = ?", (feed, old_id))
    # The done-axis travels with the entry too, and independently of the star:
    # an archived item may have no saved_entries row to carry it.
    conn.execute(
        "INSERT OR IGNORE INTO archived_entries (feed_url, entry_id, archived_at) "
        "SELECT ?, ?, archived_at FROM archived_entries WHERE feed_url = ? AND entry_id = ?",
        (feed, new_id, feed, old_id),
    )
    conn.execute("DELETE FROM archived_entries WHERE feed_url = ? AND entry_id = ?", (feed, old_id))
    if src.read:
        reader.mark_entry_as_read((feed, new_id))
    conn.execute("DELETE FROM entry_link_overrides WHERE feed_url = ? AND entry_id = ?", (feed, old_id))
    conn.commit()
    starred_archive_service.rekey_archive(feed, old_id, feed, new_id)
    _hard_delete_entry(reader, feed, old_id, src)
    return "migrated"


def migrate_feed_host_rewrite(feed_url: str, host_map: dict[str, str]) -> dict:
    """Rewrite existing entries of one feed whose id/link host matches *host_map*.
    The single-feed, inline counterpart of the apply_feed_url_rewrites script, so
    an Edit-Website flips old links immediately instead of on the next refresh."""
    import time as _time
    from collections import Counter

    stats: Counter[str] = Counter()
    reader = get_reader()
    with get_meta_connection() as conn:
        conn.execute("PRAGMA busy_timeout = 20000")
        for e in list(reader.get_entries(feed=feed_url)):
            old_id = str(e.id)
            new_id = _swap_host_in_url(old_id, host_map)
            if new_id == old_id:
                continue
            stats["match"] += 1
            new_link = _swap_host_in_url(str(e.link or old_id), host_map)
            for attempt in range(4):
                try:
                    stats[migrate_entry_to_new_host(reader, conn, feed_url, old_id, new_id, new_link)] += 1
                    break
                except sqlite3.OperationalError as exc:
                    if "locked" in str(exc).lower() and attempt < 3:
                        _time.sleep(1.5)
                        continue
                    stats["locked"] += 1
                    break
    return dict(stats)


def _normalize_alias_host(value: str) -> str:
    """Accept a pasted URL or a bare hostname, return the bare host.

    Users copy what's in the address bar, so ``https://tushar.bio/`` has to work
    as readily as ``tushar.bio``. A leading ``www.`` is dropped to match
    get_dedupe_host_aliases, which stores and compares keys without it."""
    host = (value or "").strip().lower()
    if not host:
        return ""
    if "//" in host:
        host = urlparse(host).netloc or ""
    host = host.split("@")[-1].split(":")[0].strip("/")
    host = host.removeprefix("www.")
    # A host has no path or spaces and needs at least one dot to be a domain.
    if not host or "/" in host or " " in host or "." not in host:
        return ""
    return host


_MOVE_BATCH_CAP = 500


def _rescue_unread_entries(reader, remove_url: str, keep_url: str) -> int:
    """Mark read entries in keep_url as unread when the removed feed had them unread.

    Collects URL slugs (last path segment) of unread entries in remove_url, then
    finds read entries in keep_url with matching slugs and marks them unread.
    Slug matching is domain-agnostic, so old.reddit.com vs www.reddit.com works.
    Returns the number of entries rescued.
    """
    unread_slugs: set[str] = set()
    try:
        for entry in reader.get_entries(feed=remove_url, read=False):
            slug = entry_url_slug(entry.link) if entry.link else None
            if slug:
                unread_slugs.add(slug)
    except Exception:
        LOGGER.exception("[dedup] error collecting unread slugs from %s", remove_url)
        return 0

    if not unread_slugs:
        return 0

    rescued = 0
    try:
        for entry in reader.get_entries(feed=keep_url, read=True):
            slug = entry_url_slug(entry.link) if entry.link else None
            if slug and slug in unread_slugs:
                reader.mark_entry_as_unread(entry)
                rescued += 1
    except Exception:
        LOGGER.exception("[dedup] error rescuing unread entries in %s", keep_url)

    if rescued:
        LOGGER.info("[dedup] rescued %d unread entries in %s (removed %s)", rescued, keep_url, remove_url)
    return rescued


# /feeds/deduplicate (bulk same_folder/cross_folder/upgrade auto-apply) was
# retired 2026-08-10: every duplicate-scan tier now runs through the same
# per-group Compare-then-/feeds/combine flow (see dedup-inline-results in
# app.js), so nothing calls this anymore. Its bulk apply auto-processed every
# same_folder pair unconditionally regardless of what the UI showed selected
# -- a landmine now that Compare-before-merge is the rule for every tier.

# _saved_dup_groups and the two _SAVED_DUP_BODY_* constants stay here (rather
# than moving to routes/saved.py with their only route caller,
# get_saved_duplicates) because scripts/measure_cross_feed_duplicates.py and
# scripts/merge_saved_vs_real_duplicates.py both call them as `main.<name>`.

_SAVED_DUP_BODY_HEAD_CHARS = 400
_SAVED_DUP_BODY_SQL_CHARS = 2000


def _saved_dup_groups(records: list[dict], keys: tuple[str, ...]) -> list[list[dict]]:
    """Union-find over shared signal values: records that share any non-empty
    value for any of *keys* end up in the same group. Returns groups of 2+."""
    parent = list(range(len(records)))

    def _find(i: int) -> int:
        while parent[i] != i:
            parent[i] = parent[parent[i]]
            i = parent[i]
        return i

    first_seen: dict[tuple[str, str], int] = {}
    for i, r in enumerate(records):
        for key in keys:
            val = r[key]
            if not val:
                continue
            bucket = (key, val)
            if bucket in first_seen:
                ri, rj = _find(first_seen[bucket]), _find(i)
                if ri != rj:
                    parent[rj] = ri
            else:
                first_seen[bucket] = i

    groups: dict[int, list[dict]] = {}
    for i, r in enumerate(records):
        groups.setdefault(_find(i), []).append(r)
    return [g for g in groups.values() if len(g) > 1]


def _saved_dup_host_slug(link: str, host_aliases: dict[str, str] | None = None) -> str | None:
    """Host-scoped slug key for the saved scan's confirmed tier.

    `_safe_dedup_entry_slug` is host-blind — it returns the last path segment
    and nothing else. That is correct for its other callers (per-feed churn
    history, and the multi-signal dedup where a slug alone is never one of
    `_SAFE_DEDUP_COMBOS`), but here a bare slug match *is* a confirmed
    duplicate, so unrelated publishers writing about one topic collided:
    guitarworld.com and guitarmasterclass.net each have a `pinch-harmonics`
    article, and the scan offered to delete one of them. A descriptive slug
    clears every length/hyphen guard precisely *because* it describes a topic.

    Scoping to the host keeps the tier's real job — the same article under a
    changed path on one site — and leaves genuine cross-host syndication to the
    title/body tier, which is much stronger evidence than a shared topic word.
    """
    slug = _safe_dedup_entry_slug(link)
    if not slug:
        return None
    canon = normalize_entry_link_for_dedupe(link, host_aliases)
    host = canon.partition("/")[0] if canon else ""
    return f"{host}/{slug}" if host else None


_SAVED_DUP_CHECK_TIMEOUT = 8.0


_SOFT_404_PATH_NAMES = frozenset(
    {
        "",
        "index",
        "index.html",
        "index.php",
        "index.htm",
        "home",
        "default.aspx",
        "404",
        "not-found",
        "notfound",
        "error",
        "search",
        "blog",
        "news",
        "articles",
    }
)


def _normalize_probe_path(url: str) -> list[str]:
    """Path segments of *url*, lowercased, with empties and a trailing
    index-file dropped — the shape used to compare a redirect's start and end."""
    try:
        path = urlparse(url).path or "/"
    except ValueError:
        return []
    segs = [s.lower() for s in path.split("/") if s]
    if segs and segs[-1] in {"index.html", "index.htm", "index.php", "default.aspx"}:
        segs.pop()
    return segs


def _looks_like_soft_404(original: str, final: str) -> bool:
    """True if a 200 response looks like the article is gone and the site
    quietly redirected to an index instead.

    `_check_saved_url` only counts 404/410, so this whole class reads as alive:
    probing 8 guitarplayer.com articles returned 200 for all of them while 4 had
    been redirected to the bare `/lessons` index. The article is gone; the site
    just won't say so.

    Deliberately narrow — it fires only when a redirect *lost* path depth, i.e.
    the destination is an ancestor of where we started (or the root, or an
    obviously index-ish name). A redirect that merely reshapes a URL at the same
    depth (`/2019/post-name` → `/blog/post-name`) is a site reorganization, not
    a soft 404, and must not trip this: it is the case where the article is
    still there. Same-page redirects (http→https, adding `www`, adding a
    trailing slash) never count, since the path is unchanged.
    """
    try:
        o_host = (urlparse(original).netloc or "").lower().removeprefix("www.")
        f_host = (urlparse(final).netloc or "").lower().removeprefix("www.")
    except ValueError:
        return False
    # A cross-domain redirect is a migration or a parking page — too ambiguous
    # to call from the URL alone, and not what this is for.
    if o_host != f_host:
        return False

    o_segs = _normalize_probe_path(original)
    f_segs = _normalize_probe_path(final)
    # Needs an article-shaped starting point (2+ segments) to have lost
    # anything: probing a section page says nothing about a missing article.
    if len(o_segs) < 2 or f_segs == o_segs:
        return False
    if len(f_segs) >= len(o_segs):
        return False  # same depth or deeper — a reshape, not a disappearance
    # Landed on the root.
    if not f_segs:
        return True
    # Collapsed onto a top-level section page. This is the observed shape and
    # the ancestor test misses it, because sites redirect *across* sections:
    # guitarplayer.com sends /technique/<article> to /lessons, so the
    # destination shares no prefix with the original and isn't named like an
    # index either. A single remaining segment on the same host is a section
    # landing page, not an article.
    if len(f_segs) == 1:
        return True
    # Deeper destination that is still a strict ancestor, or index-ish.
    return o_segs[: len(f_segs)] == f_segs or f_segs[-1] in _SOFT_404_PATH_NAMES


def _check_saved_url(url: str) -> dict:
    """Liveness probe for one saved article URL. Honest UA, SSRF-guarded,
    redirects followed with per-hop validation. Never raises.

    Classification is deliberately conservative: only 404/410 count as dead
    (the keeper auto-flip acts on it); 403/429/5xx are bot-walls or hiccups,
    not proof the page is gone. A HEAD failure is retried once as a GET —
    some servers reject or mishandle HEAD but serve the page fine.

    `soft_dead` is reported separately and never folded into `dead`: it is a
    URL-shape heuristic (see `_looks_like_soft_404`), and `dead` is what arms a
    deletion. Same rule as the dupe scan's probe arming — a guess may inform the
    user, never pre-check a destructive box."""
    headers = {"User-Agent": LECTIO_HONEST_USER_AGENT}
    status: int | None = None
    final_url = url
    try:
        resp = url_guard.safe_head_follow(url, timeout=_SAVED_DUP_CHECK_TIMEOUT, headers=headers)
        status, final_url = resp.status_code, str(resp.url)
        if status >= 400:
            with url_guard.build_client(timeout=_SAVED_DUP_CHECK_TIMEOUT, headers=headers) as client:
                resp = url_guard.safe_get(client, url)
            status, final_url = resp.status_code, str(resp.url)
    except url_guard.UnsafeURLError:
        return {"status": None, "alive": False, "dead": False, "soft_dead": False, "final_url": url, "error": "unsafe URL"}
    except Exception as exc:  # noqa: BLE001 — DNS failure, timeout, TLS, ...
        return {"status": None, "alive": False, "dead": False, "soft_dead": False, "final_url": url, "error": type(exc).__name__}
    return {
        "status": status,
        "alive": status < 400,
        "dead": status in (404, 410),
        "soft_dead": status < 400 and _looks_like_soft_404(url, final_url),
        "final_url": final_url,
        "error": None,
    }


def _autofile_excluded_targets(feed_urls: Iterable[str], conn=None) -> frozenset[str]:
    """Feeds an unfiled saved article must never be filed into.

    Saved Articles itself, obviously — and every YouTube feed. A saved *page* is
    never really a video-channel post, and channels routinely share a name with
    the blog they accompany, so with only titles on screen a YouTube feed is
    exactly the target you'd pick by mistake. Cheap insurance: it only ever
    triggers for a saved youtube.com link, where filing into the channel feed
    would be wrong anyway.

    Plus anything the user has marked in `non_feed_subscriptions`: a single
    article URL that ended up subscribed as a feed is a plausible-looking target
    on the right host, and filing that site's backlog into a one-article stub is
    exactly the wrong outcome.
    """
    excluded = {saved_articles_service.SAVED_FEED_URL}
    excluded.update(u for u in feed_urls if "youtube.com/feeds/videos.xml" in str(u))
    if conn is not None:
        excluded.update(r[0] for r in conn.execute("SELECT feed_url FROM non_feed_subscriptions"))
    return frozenset(excluded)


# _current_autofile_plan stays here (rather than moving to routes/saved.py with
# preview_saved_autofile, its only route caller) because routes/system.py's
# Instapaper import also calls it, to report what a fresh import could
# auto-file without re-scanning the whole Saved Articles backlog.
def _current_autofile_plan(restrict_to: set[str] | None = None) -> tuple[list[dict], list[dict], dict]:
    """Assemble the autofile plan for the current user.

    Returns ``(plan, marked, titles)`` — the worklist, the clusters whose host
    the user has marked "not a feed" (settled business, reported separately),
    and feed_url → display title.

    *restrict_to* limits the saved articles considered to those entry ids, which
    is how the Instapaper importer reports what a fresh import could file
    without re-scanning the whole backlog.
    """
    saved_url = saved_articles_service.SAVED_FEED_URL
    with get_reader() as reader:
        if reader.get_feed(saved_url, None) is None:
            return [], [], {}
        titles = {}
        # A feed declares its host twice: its own URL, and the site it
        # advertises. Both count — a FeedBurner feed's URL host says nothing
        # about the site whose posts it carries, but its link does.
        declared_hosts: dict[str, set[str]] = {}
        for f in reader.get_feeds():
            url = str(f.url)
            titles[url] = f.user_title or f.title or url
            hosts = {saved_autofile_service.article_host(url)}
            if getattr(f, "link", None):
                hosts.add(saved_autofile_service.article_host(str(f.link)))
            declared_hosts[url] = {h for h in hosts if h}
        db = reader._storage.get_db()
        feed_sizes = {str(f): int(n) for f, n in db.execute("SELECT feed, COUNT(*) FROM entries GROUP BY feed")}
        # Only genuinely-kept saves are worth filing; a read, unstarred row is a
        # husk from an older move that predates source cleanup.
        with get_meta_connection() as conn:
            kept = {r[0] for r in conn.execute("SELECT entry_id FROM saved_entries WHERE feed_url = ?", (saved_url,))}
            non_feed = {r[0] for r in conn.execute("SELECT host FROM autofile_non_feed_hosts")}
        saved_rows = [
            (str(i), str(link or i))
            for i, link in db.execute("SELECT id, link FROM entries WHERE feed = ?", (saved_url,))
            if str(i) in kept and (restrict_to is None or str(i) in restrict_to)
        ]
        feed_links = [
            (str(f), str(link))
            for f, link in db.execute(
                "SELECT feed, link FROM entries WHERE link IS NOT NULL AND feed != ?",
                (saved_url,),
            )
        ]

    with get_meta_connection() as conn:
        excluded = _autofile_excluded_targets(titles, conn)
    plan = saved_autofile_service.build_autofile_plan(
        saved_rows,
        feed_links,
        feed_titles=titles,
        feed_hosts=declared_hosts,
        feed_sizes=feed_sizes,
        exclude_feeds=excluded,
    )
    # Hosts marked "not a feed" are settled business — their saves are genuine
    # one-off captures, so they drop out of the worklist entirely rather than
    # reappearing unresolved on every pass. Reported so they can be reviewed.
    marked = [c for c in plan if c["host"] in non_feed]
    plan = [c for c in plan if c["host"] not in non_feed]
    return plan, marked, titles


def _current_unstar_tagged_plan(keep_tags: set[str] | None = None) -> dict:
    """Assemble the unstar-tagged plan for the current user.

    Reads the star rows, every manual tag, and the archived-at set, and hands
    them to the pure decision layer. Read-only.
    """
    with get_reader() as reader:
        db = reader._storage.get_db()
        tags_by_key: dict[tuple[str, str], list[str]] = {}
        for feed, eid, key in db.execute(
            "SELECT feed, id, key FROM entry_tags WHERE key LIKE ?",
            (f"{MANUAL_TAG_KEY_PREFIX}%",),
        ):
            tags_by_key.setdefault((str(feed), str(eid)), []).append(str(key)[len(MANUAL_TAG_KEY_PREFIX) :])
    with get_meta_connection() as conn:
        starred = {(str(f), str(e)) for f, e in conn.execute("SELECT feed_url, entry_id FROM saved_entries")}
    plan = unstar_tagged_service.build_unstar_plan(starred, tags_by_key, keep_tags=keep_tags)
    plan["queue_like_tags"] = unstar_tagged_service.queue_like_tags({row["tag"] for row in plan["per_tag"]})
    return plan


@app.post("/refresh")
def refresh(
    folder_id: int = Form(...),
    tag: str | None = Form(default=None),
    sort_by: str | None = Form(default=None),
    sort_dir: str | None = Form(default=None),
    read_filter: str | None = Form(default=None),
    star_only: str | None = Form(default=None),
    resume_read_filter: str | None = Form(default=None),
    list_feed_url: str | None = Form(default=None),
    feed_url: str | None = Form(default=None),
    entry_id: str | None = Form(default=None),
):
    normalized_tag = normalize_tag_value(tag)
    normalized_read_filter = normalize_read_filter(read_filter)
    tag_query = f"&tag={quote_plus(normalized_tag)}" if normalized_tag else ""
    list_feed_query = f"&list_feed_url={quote_plus(list_feed_url)}" if list_feed_url else ""
    sort_query = (
        build_sort_query(sort_by, sort_dir)
        + build_read_filter_query(read_filter)
        + build_star_only_query(star_only)
        + build_resume_read_filter_query(resume_read_filter, active_read_filter=normalized_read_filter)
    )
    entry_query = ""
    if feed_url and entry_id:
        entry_query = _entry_query_suffix(feed_url, entry_id)
    retry_after_seconds = check_and_mark_manual_refresh()
    if retry_after_seconds > 0:
        return RedirectResponse(
            url=(
                local_url(
                    f"/?folder_id={folder_id}{list_feed_query}{tag_query}{sort_query}{entry_query}&message="
                    f"{quote_plus(f'Refresh throttled. Try again in {retry_after_seconds}s.')}"
                )
            ),
            status_code=303,
        )

    with get_meta_connection() as conn:
        feed_urls = get_folder_feed_urls(conn, folder_id)
    if REFRESH_DEBUG_ENABLED:
        LOGGER.info(
            "[refresh] manual folder refresh: folder_id=%s feed_count=%d list_feed_url=%s tag=%s",
            folder_id,
            len(feed_urls),
            list_feed_url,
            normalized_tag,
        )
    job = _folder_refresh_job_state(create=True)
    with _folder_refresh_jobs_lock:
        if job.get("running"):
            return RedirectResponse(
                url=local_url(
                    f"/?folder_id={folder_id}{list_feed_query}{tag_query}{sort_query}{entry_query}&message="
                    f"{quote_plus('A refresh is already running.')}"
                ),
                status_code=303,
            )
        job.update({"running": True, "done": False, "job_id": secrets.token_hex(8), "total": len(feed_urls), "finished_at": None})
    uid = tenancy.current_user_id()
    try:
        threading.Thread(target=_run_in_user_context, args=(uid, _run_folder_refresh, list(feed_urls), job), daemon=True).start()
    except Exception:
        with _folder_refresh_jobs_lock:
            job["running"] = False  # the worker's own finally never ran, so the job would read as running forever
        raise
    return RedirectResponse(
        url=(
            local_url(
                f"/?folder_id={folder_id}{list_feed_query}{tag_query}{sort_query}{entry_query}&message="
                f"{quote_plus(f'Refreshing {len(feed_urls)} feeds…')}"
            )
        ),
        status_code=303,
    )


# One folder refresh at a time per user. A big folder (hundreds of feeds) takes minutes, longer than a reverse proxy waits, so the
# request only starts the job; the page polls /api/refresh/status and reloads when it finishes.
_folder_refresh_jobs = _PerUserDict()
_folder_refresh_jobs_lock = threading.Lock()


@overload
def _folder_refresh_job_state(create: Literal[True]) -> dict: ...
@overload
def _folder_refresh_job_state(create: bool = False) -> dict | None: ...
def _folder_refresh_job_state(create: bool = False) -> dict | None:
    with _folder_refresh_jobs_lock:
        job = _folder_refresh_jobs.get("job")
        if job is None and create:
            job = {"running": False}
            _folder_refresh_jobs["job"] = job
        return job


def _run_folder_refresh(feed_urls: list[str], job: dict) -> None:
    try:
        with get_meta_connection() as conn:
            scraper_service.refresh_all_scraped_feeds(conn)
            devto_service.refresh_all_devto_feeds(conn)
            # Only touch DeviantArt when it's actually configured — avoids needless work
            # (and the access-token settings lookup) for the common no-DA setup.
            _da_cid, _da_secret = get_deviantart_credentials()
            if _da_cid and _da_secret:
                deviantart_service.refresh_all_deviantart_feeds(conn, _da_cid, _da_secret, access_token=get_deviantart_user_token())
        feed_refresh_service.update_feeds(feed_urls, enhance=False, inline_followups=True)
        _run_automation_after_refresh(set(feed_urls))
        invalidate_unread_counts_cache()
        # Lead images / durations are network-heavy; fetch them off this thread too.
        _spawn_feed_enhancement(feed_urls)
    except Exception:  # noqa: BLE001
        LOGGER.exception("[refresh] background folder refresh failed")
    finally:
        with _folder_refresh_jobs_lock:
            job.update({"running": False, "done": True, "finished_at": time.time()})


@app.get("/api/refresh/status")
def refresh_status():
    job = _folder_refresh_job_state()
    if job is None:
        return JSONResponse({"ok": True, "running": False})
    with _folder_refresh_jobs_lock:
        snap = dict(job)
    finished_at = snap.get("finished_at")
    return JSONResponse(
        {
            "ok": True,
            "running": bool(snap.get("running")),
            "job_id": snap.get("job_id"),
            "total": snap.get("total"),
            "age_seconds": None if finished_at is None else max(0, int(time.time() - finished_at)),
        }
    )


@app.post("/refresh/feed")
def refresh_feed(
    folder_id: int = Form(...),
    feed_url: str = Form(...),
    list_feed_url: str | None = Form(default=None),
    tag: str | None = Form(default=None),
    sort_by: str | None = Form(default=None),
    sort_dir: str | None = Form(default=None),
    read_filter: str | None = Form(default=None),
    star_only: str | None = Form(default=None),
    resume_read_filter: str | None = Form(default=None),
    entry_id: str | None = Form(default=None),
):
    normalized_tag = normalize_tag_value(tag)
    normalized_read_filter = normalize_read_filter(read_filter)
    retry_after_seconds = 0 if _is_local_dev_feed(feed_url) else check_and_mark_manual_refresh()
    list_feed_query = f"&list_feed_url={quote_plus(list_feed_url)}" if list_feed_url else ""
    tag_query = f"&tag={quote_plus(normalized_tag)}" if normalized_tag else ""
    sort_query = (
        build_sort_query(sort_by, sort_dir)
        + build_read_filter_query(read_filter)
        + build_star_only_query(star_only)
        + build_resume_read_filter_query(resume_read_filter, active_read_filter=normalized_read_filter)
    )
    entry_query = _entry_query_suffix(feed_url, entry_id)
    if retry_after_seconds > 0:
        return RedirectResponse(
            url=(
                local_url(
                    f"/?folder_id={folder_id}{list_feed_query}{tag_query}{sort_query}{entry_query}&message="
                    f"{quote_plus(f'Refresh throttled. Try again in {retry_after_seconds}s.')}"
                )
            ),
            status_code=303,
        )

    if REFRESH_DEBUG_ENABLED:
        LOGGER.info(
            "[refresh] manual single-feed refresh: folder_id=%s feed_url=%s list_feed_url=%s tag=%s",
            folder_id,
            feed_url,
            list_feed_url,
            normalized_tag,
        )
    feed_id = scraper_service.scraped_feed_id_from_url(feed_url)
    if feed_id:
        with get_meta_connection() as conn:
            scraper_service.refresh_scraped_feed_by_id(conn, feed_id)
    # A deliberate click on one feed must actually attempt a fetch, not silently
    # no-op behind a backoff window the user has no visibility into — see
    # FeedRefreshService.update_feeds' bypass_backoff docstring.
    skipped = feed_refresh_service.update_feeds([feed_url], enhance=False, bypass_backoff=True)
    if skipped:
        # Another pass (a folder refresh, the scheduler) is updating this feed right now; a second concurrent fetch would collide
        # with it, so say so rather than report a refresh that didn't happen.
        return RedirectResponse(
            url=local_url(
                f"/?folder_id={folder_id}{list_feed_query}{tag_query}{sort_query}{entry_query}"
                f"&message={quote_plus('This feed is already being refreshed; try again in a moment.')}"
            ),
            status_code=303,
        )
    _run_automation_after_refresh({feed_url})
    invalidate_unread_counts_cache()
    _spawn_feed_enhancement([feed_url])
    return RedirectResponse(
        url=(
            local_url(
                f"/?folder_id={folder_id}{list_feed_query}{tag_query}{sort_query}{entry_query}"
                f"&message={quote_plus('Feed refresh complete.')}"
            )
        ),
        status_code=303,
    )


def _run_on_star_destinations(feed_url: str, entry_id: str) -> None:
    """Fire the configured "on star" destinations for a freshly-starred entry.
    One-way (never on unstar), best-effort; runs in a background thread."""
    try:
        send_ip = get_runtime_setting(SETTING_STAR_SEND_INSTAPAPER, "0") == "1"
        playlist_id = (get_runtime_setting(SETTING_STAR_SEND_YT_PLAYLIST) or "").strip()
        email_to = (get_runtime_setting(SETTING_STAR_SEND_EMAIL) or "").strip()
        send_quire = get_runtime_setting(SETTING_STAR_SEND_QUIRE, "0") == "1" and is_quire_configured()
        if not (send_ip or playlist_id or email_to or send_quire):
            return
        with get_reader() as reader:
            entry = reader.get_entry((feed_url, entry_id), None)
            feed_title = ""
            if entry:
                try:
                    feed_title = str(getattr(reader.get_feed(feed_url), "title", None) or "")
                except Exception:
                    feed_title = ""
        if not entry:
            return
        link = str(getattr(entry, "link", "") or "")
        title = str(getattr(entry, "title", "") or "")

        if send_ip and is_instapaper_configured() and link:
            u = get_runtime_setting(SETTING_INSTAPAPER_USERNAME).strip()
            p = get_runtime_setting(SETTING_INSTAPAPER_PASSWORD).strip()
            if u and p:
                ok, err = _instapaper_save_url(u, p, link, title)
                if not ok:
                    LOGGER.warning("[on-star] instapaper save failed: %s", err)

        if playlist_id:
            token = get_youtube_oauth_token()
            if token:
                own_vid = youtube_embeds.own_video_id(feed_url, link)
                for vid in [own_vid] if own_vid else []:
                    try:
                        if youtube_oauth_service.video_in_playlist(token, playlist_id, vid):
                            continue  # already there (the API would add a second copy)
                        youtube_oauth_service.add_video_to_playlist(token, playlist_id, vid)
                    except youtube_oauth_service.QuotaExceeded:
                        mark_yt_quota_exhausted()
                        break
                    except Exception as exc:  # noqa: BLE001
                        LOGGER.warning("[on-star] playlist add failed for %s: %s", vid, exc)

        if email_to and is_email_configured() and link:
            try:
                ok, err = send_article_email(
                    get_resend_api_key(),
                    get_resend_from(),
                    email_to,
                    title,
                    feed_title,
                    link,
                    _get_entry_excerpt(entry),
                    **email_article_extras(feed_url, entry_id, entry),
                )
                if not ok:
                    LOGGER.warning("[on-star] email failed: %s", err)
            except Exception as exc:  # noqa: BLE001
                LOGGER.warning("[on-star] email failed: %s", exc)

        if send_quire and link:
            if get_quire_usage_status()["state"] == "blocked":
                LOGGER.warning("[on-star] quire skipped: rate limit reached")
            else:
                token = get_quire_user_token()
                if token:
                    ok, err = _quire_add_entry(token, quire_project_oid(), title, link, feed_title)
                    if not ok:
                        LOGGER.warning("[on-star] quire add failed: %s", err)

        reddit_sub = (get_runtime_setting(SETTING_STAR_SEND_REDDIT_SUBREDDIT) or "").strip().removeprefix("r/").strip("/")
        if reddit_sub and link:
            reddit_token = get_reddit_user_token()
            if reddit_token:
                try:
                    reddit_service.submit_link(reddit_token, reddit_sub, title, link)
                except Exception as exc:  # noqa: BLE001
                    LOGGER.warning("[on-star] reddit submit failed: %s", exc)
    except Exception:
        LOGGER.exception("[on-star] error for %s/%s", feed_url, entry_id)


# --- Save Article (read-later capture of arbitrary URLs) ---


def _resolve_redirector_url(url: str) -> str:
    """Follow a feed-redirector link (feedproxy/feedburner) to its real
    destination before saving, so the stored article never depends on the
    redirector staying alive. SSRF-guarded; returns the input on failure."""
    try:
        with url_guard.build_client(timeout=8.0, headers={"User-Agent": READABILITY_USER_AGENT}) as client:
            resp = url_guard.safe_get(client, url, headers={"User-Agent": READABILITY_USER_AGENT})
        final = str(resp.url)
        if final and not link_canonical.is_redirector_link(final):
            return final
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("[canonical-link] redirector resolve failed for %s: %s", url, exc)
    return url


def _find_subscribed_entry_for_url(article_url: str) -> tuple[str, str] | None:
    """An existing entry for this article in some *other* feed, or None.

    Matched on the canonical link rather than the id, because the two rarely
    agree: Medium's feed uses a `/p/<hash>` guid while the article URL is the
    long slug, yet both entries carry the same `link`. Declared domain
    migrations fold in through get_dedupe_host_aliases, so an author's old host
    still pairs with their current one.

    Prefers a feed-provided entry: it keeps updating and it is the one carrying
    the publisher's tags, which is precisely what a capture cannot supply.
    """
    aliases = get_dedupe_host_aliases()
    target = normalize_entry_link_for_dedupe(article_url, aliases)
    if not target:
        return None
    # Candidate spellings for an indexed equality match; the normalized compare
    # below is what actually decides, so this only has to be generous enough.
    bare = article_url.split("#")[0].rstrip("/")
    variants = {bare, bare + "/"}
    for scheme_a, scheme_b in (("https://", "http://"), ("http://", "https://")):
        if bare.startswith(scheme_a):
            swapped = scheme_b + bare[len(scheme_a) :]
            variants |= {swapped, swapped + "/"}
    for v in list(variants):
        if "://www." in v:
            variants.add(v.replace("://www.", "://", 1))
        else:
            variants.add(v.replace("://", "://www.", 1))
    try:
        conn = sqlite3.connect(str(tenancy.reader_db_path()), timeout=5.0)
        try:
            placeholders = ",".join("?" for _ in variants)
            rows = conn.execute(
                f"SELECT feed, id, link, added_by FROM entries WHERE link IN ({placeholders})",
                list(variants),
            ).fetchall()
        finally:
            conn.close()
    except Exception:  # noqa: BLE001 — never block a save on the lookup
        LOGGER.exception("[save] subscribed-entry lookup failed for %s", article_url)
        return None

    best: tuple[str, str] | None = None
    for feed, entry_id, link, added_by in rows:
        if str(feed) == saved_articles_service.SAVED_FEED_URL:
            continue  # its own saved copy is handled by the normal duplicate path
        if normalize_entry_link_for_dedupe(str(link or ""), aliases) != target:
            continue
        if str(added_by) == "feed":
            return (str(feed), str(entry_id))  # the survivor we want
        best = best or (str(feed), str(entry_id))
    return best


def _save_article_for_current_user(url: str, extract=None, refresh_content: bool = False) -> dict:
    """Run the saved-articles service with the current tenancy's reader/meta
    DB, wiring in readability extraction and the starred-archive enqueue.
    *extract* overrides the server-side fetch+extract — e.g. extraction from
    browser-captured HTML for the extension save protocol; those captured
    saves also refresh_content on duplicates (a deliberate re-capture)."""
    if link_canonical.is_redirector_link(url):
        url = _resolve_redirector_url(url)
    reader = get_reader()
    with get_meta_connection() as conn:
        result = saved_articles_service.save_article(
            reader,
            conn,
            url,
            extract=extract or fetch_readability_article,
            enqueue_archive=starred_archive_service.enqueue_archive,
            refresh_content=refresh_content,
            find_existing_entry=_find_subscribed_entry_for_url,
        )
    if result.get("created_feed"):
        # The Saved Articles feed just appeared — surface it in the sidebar tree.
        invalidate_meta_structure_cache()
        invalidate_problematic_feeds_cache()
    # Invalidate on a new save OR a resurface: a resurface un-archives and marks
    # a *duplicate* unread, which changes the unread counts and the Saved Inbox
    # list — gating only on "not duplicate" left those stale, so a re-saved
    # article never appeared in the Inbox until the cache expired.
    if result.get("ok") and (not result.get("duplicate") or result.get("resurfaced")):
        invalidate_unread_counts_cache()
    return result


# The wire value that selects whole-page capture over readability extraction,
# on both /articles/save and /articles/refresh-content. Named because it is
# compared in two places here and sent from two more in static/js/app.js — a
# typo on any of them silently falls back to readability rather than erroring.
CAPTURE_MODE_FULL = "full"
# Re-fetch the Internet Archive's snapshot instead of the live page. Same
# reasoning about the spelling being shared with the client.
CAPTURE_MODE_ARCHIVE = "archive"


def _entry_source_url(feed_url: str, entry_id: str) -> str | None:
    """The address a capture is re-fetched FROM: its current link, falling back
    to its id (a capture's id is the address it was first saved from)."""
    try:
        with get_reader() as reader:
            entry = reader.get_entry((feed_url, entry_id), None)
    except Exception:  # noqa: BLE001
        return None
    if entry is None:
        return None
    link = str(getattr(entry, "link", "") or "")
    if link.startswith(("http://", "https://")):
        return link
    return entry_id if entry_id.startswith(("http://", "https://")) else None


def _autofetch_host_in_cooldown(host: str) -> bool:
    if not host:
        return False
    with _autofetch_hosts_lock:
        until = _autofetch_failed_hosts.get(host)
        if until is None:
            return False
        if time.time() >= until:
            del _autofetch_failed_hosts[host]
            return False
        return True


def _mark_autofetch_host_failed(host: str) -> None:
    if not host:
        return
    with _autofetch_hosts_lock:
        _autofetch_failed_hosts[host] = time.time() + _AUTOFETCH_HOST_COOLDOWN_S
        # Bounded: this is a politeness memo, not a record.
        if len(_autofetch_failed_hosts) > 512:
            for stale in sorted(_autofetch_failed_hosts, key=lambda h: _autofetch_failed_hosts[h])[:128]:
                del _autofetch_failed_hosts[stale]


# Entries whose automatic re-fetch came back no richer than the stored copy, so a later star or tag edit doesn't send the same fetch again.
_AUTOFETCH_NOT_RICHER_TTL_S = 24 * 3600
_autofetch_not_richer: dict[tuple[str, str], float] = {}


def _autofetch_known_not_richer(feed_url: str, entry_id: str) -> bool:
    with _autofetch_jobs_lock:
        until = _autofetch_not_richer.get((feed_url, entry_id))
        if until is None:
            return False
        if time.monotonic() >= until:
            del _autofetch_not_richer[(feed_url, entry_id)]
            return False
        return True


def _autofetch_note_not_richer(feed_url: str, entry_id: str) -> None:
    with _autofetch_jobs_lock:
        _autofetch_not_richer[(feed_url, entry_id)] = time.monotonic() + _AUTOFETCH_NOT_RICHER_TTL_S
        if len(_autofetch_not_richer) > 2048:  # a memo, not a record
            for stale in sorted(_autofetch_not_richer, key=lambda k: _autofetch_not_richer[k])[:512]:
                del _autofetch_not_richer[stale]


def _autofetch_is_richer(candidate: str, stored: str) -> bool:
    """Is an automatic re-fetch worth replacing the stored body with? By text, not by image count.

    ``_reader_copy_is_richer`` leads with images, which is right for a picture post but wrong here: the extractor prepends the page's
    og:image to what it finds, so a site blurb with a share card would beat a short text post, and a teaser carrying tracking pixels
    would beat the full article. More text wins; equal text needs more images."""
    cand, cur = _reader_text_length(candidate), _reader_text_length(stored)
    return cand > cur or (cand == cur and _reader_img_count(candidate) > _reader_img_count(stored))


def _autofetch_prune_stale_jobs() -> None:
    now = time.monotonic()
    for key in list(_autofetch_jobs.keys()):
        job = _autofetch_jobs.get(key)
        if job and not job.get("running") and (now - (job.get("finished_at") or now)) > _AUTOFETCH_JOB_STALE_S:
            _autofetch_jobs.pop(key, None)


# A feed's "Read <title> on <site>." / "The post … appeared first on …" trailer after an excerpt cut off with an ellipsis.
_TEASER_TRAILER_RE = re.compile(
    r"(?:\.{3}|\u2026)\s*(?:\[\s*(?:\.{3}|\u2026)\s*\]\s*)?(?:Read\b.{0,300}?\bon\b.{0,200}?|The post\b.{0,300}?\bon\b.{0,200}?)\.?\s*$",
    re.IGNORECASE | re.DOTALL,
)


def _is_truncated_teaser(html_text: str) -> bool:
    """An excerpt cut off mid-thought plus the feed's read-more trailer: the trailer alone can push it past the plausibility
    floor (Bass Musician Magazine: 446 characters, 79 of them the trailer), so the keep-time re-fetch skipped a stub."""
    if not html_text:
        return False
    text = re.sub(r"\s+", " ", _READER_TEXT_TAG_RE.sub(" ", html_text)).strip()
    return bool(_TEASER_TRAILER_RE.search(text))


def _maybe_autofetch_on_keep(feed_url: str, entry_id: str) -> bool:
    """Re-fetch a *stub* article in the background when it is starred or tagged.

    Keeping something is the moment you find out its feed only ever shipped a
    teaser. This closes that gap — but only for teasers, and that restraint is
    the whole design:

    - **Only when the stored copy is thin.** ``_archived_copy_is_plausible`` is
      the same test the reader uses to reject a failed extraction, so "not
      plausible" here means summary-only or truncated. A feed that ships full
      posts is left alone, because re-fetching one can only make it worse: the
      live page may now be a paywall, a cookie wall, a 404, or a readability
      miss that locks onto a sidebar. Overwriting a good copy at the exact moment
      the reader marked it worth keeping is the failure to avoid.
    - **Only from a single, user-initiated action.** This is called from the star
      and tag ROUTES, never from ``set_manual_tags_for_entry`` — which the feed
      auto-taggers also drive, at ingest, across everything a feed just
      delivered. Hooking the service would turn one refresh into a burst of
      outbound requests at one host.
    - **Never for a Lectio capture.** Those were fetched from the page already;
      there is no feed teaser to improve on.

    The offline archive capture (``enqueue_archive``) already fires on both star
    and tag and is what Read Mode reads. This is the other half: it replaces the
    stub in the *entry pane*, which shows stored feed content.

    Returns True when a background re-fetch is (now or already) in flight for
    this entry, so the calling star/tag route can tell the client an open pane
    on this entry is worth polling — see /entries/autofetch-status above.
    """
    try:
        if saved_articles_service.is_saved_articles_feed(feed_url):
            return False
        with get_reader() as reader:
            entry = reader.get_entry((feed_url, entry_id), None)
        if entry is None or not (entry.link or "").startswith(("http://", "https://")):
            return False
        # Richest element, not content[0]: a feed can lead with a stub (guitarworld ships a literal "null" text/plain first)
        # and that made a full article look thin, so tagging it overwrote it with a worse extraction.
        _best = _richest_content(entry, entry.content[0] if entry.content else None)
        stored = (_best.value if _best is not None else None) or entry.summary or ""
        if _archived_copy_is_plausible(stored) and not _is_truncated_teaser(stored):
            return False  # a real article already — leave it alone
        if _autofetch_known_not_richer(feed_url, entry_id):
            return False  # the last fetch of this page was no better than what's stored
        host = urlparse(entry.link).netloc.lower()
        if _autofetch_host_in_cooldown(host):
            return False  # this host already refused us; don't keep asking
    except Exception:  # noqa: BLE001 — never let this break the star/tag itself
        LOGGER.debug("auto-refetch precheck failed for %s/%s", feed_url, entry_id, exc_info=True)
        return False

    _job_key = (feed_url, entry_id)
    with _autofetch_jobs_lock:
        _existing_job = _autofetch_jobs.get(_job_key)
        if _existing_job and _existing_job.get("running"):
            return True  # already in flight for this entry — the pane can still poll it
        _autofetch_prune_stale_jobs()
        job = {"running": True, "ok": None, "started_at": time.monotonic(), "finished_at": None}
        _autofetch_jobs[_job_key] = job

    _uid = tenancy.current_user_id()

    def _work() -> None:
        try:
            result = _refresh_captured_article_for_current_user(feed_url, entry_id, only_if_richer=True)
            job["ok"] = bool(result.get("ok"))
            if result.get("ok"):
                LOGGER.info("[auto-refetch] %s/%s -> refreshed", feed_url, entry_id)
                return
            if result.get("not_richer"):
                _autofetch_note_not_richer(feed_url, entry_id)
                LOGGER.info("[auto-refetch] %s/%s -> fetched page not richer than the stored copy; left alone", feed_url, entry_id)
                return
            # A failure here is about the SITE, not this article — blocked,
            # unreachable, or serving something else entirely — so remember the
            # host rather than rediscovering it on the next tag.
            _mark_autofetch_host_failed(host)
            LOGGER.info(
                "[auto-refetch] %s/%s -> %s (pausing %s for %dh)",
                feed_url,
                entry_id,
                result.get("error", "no change"),
                host,
                _AUTOFETCH_HOST_COOLDOWN_S // 3600,
            )
        except Exception:  # noqa: BLE001
            job["ok"] = False
            _mark_autofetch_host_failed(host)
            LOGGER.warning("[auto-refetch] failed for %s/%s", feed_url, entry_id, exc_info=True)
        finally:
            job["running"] = False
            job["finished_at"] = time.monotonic()

    # Off-request so the star stays instant, and through the tenancy helper
    # because a bare thread has no user bound.
    threading.Thread(target=lambda: _run_in_user_context(_uid, _work), daemon=True).start()
    return True


def _full_content_entry_hook(uid: str, entry, is_new: bool) -> None:
    """reader NEW-entry hook: queue ingest-time page work (full-content fetch, page topics)."""
    if not is_new:
        return
    with tenancy.user_context(uid):
        full_content_fetch_service.on_entry_updated(entry, is_new)
        page_topics_service.on_entry_updated(entry, is_new)


def _spawn_full_content_drain(feed_urls: list[str]) -> None:
    """Off-thread: a single full-page fetch can take a minute (FlareSolverr rung), and the scheduled refresh calls enhancement
    in-line."""
    if not feed_urls:
        return
    uid = tenancy.current_user_id()
    # Separate threads: a page-topics fetch is a few KB and shouldn't queue behind a minute-long full-page fetch.
    for drain in (full_content_fetch_service.drain, page_topics_service.drain):
        threading.Thread(target=_run_in_user_context, args=(uid, drain, list(feed_urls), uid), daemon=True).start()


full_content_fetch_service = full_content_fetch.FullContentFetchService(
    get_meta_connection=lambda: get_meta_connection(),
    get_reader=lambda: get_reader(),
    refetch=lambda feed_url, entry_id: _refresh_captured_article_for_current_user(
        feed_url, entry_id, bump_received=False, only_if_richer=True
    ),
    is_thin=lambda html: not _archived_copy_is_plausible(html),
    host_in_cooldown=lambda host: _autofetch_host_in_cooldown(host),
    mark_host_failed=lambda host: _mark_autofetch_host_failed(host),
    is_excluded_feed=lambda feed_url: saved_articles_service.is_saved_articles_feed(feed_url),
)


def _fetch_page_prefix(url: str) -> tuple[str, int]:
    with url_guard.build_client(timeout=10.0, headers={"User-Agent": READABILITY_USER_AGENT}) as client:
        html, _final_url, status = url_guard.safe_get_prefix(client, url, max_bytes=page_topics.PREFIX_BYTES)
    return html, status


def _page_topics_escalated_fetch(url: str) -> str:
    """Full-ladder fetch for page topics; a busy shared solver slot defers the entry rather than counting against the host."""
    try:
        return page_fetcher.fetch(url, timeout=15.0).html
    except page_fetch.PageFetchError as exc:
        if exc.solver_busy:
            raise page_topics.FetchDeferred(url) from exc
        raise


page_topics_service = page_topics.PageTopicsService(
    get_meta_connection=lambda: get_meta_connection(),
    get_reader=lambda: get_reader(),
    fetch_prefix=_fetch_page_prefix,
    extract_tags=lambda html, url: feed_tags_service_mod.extract_page_tags(html, url),
    record_tags=lambda feed_url, entry_id, tags: feed_tag_service.record_entry_tags(feed_url, [(entry_id, tags)], source="page"),
    is_excluded_feed=lambda feed_url: saved_articles_service.is_saved_articles_feed(feed_url),
    fetch_escalated=lambda url: _page_topics_escalated_fetch(url),
    on_tags_recorded=lambda feed_url: _run_tag_filter_rules_for_feed(feed_url),
)


def _scraped_feed_content_selector(feed_url: str) -> str:
    """The per-feed content_selector of a scraped (file://) feed, or "" when there is none."""
    feed_id = scraper_service.scraped_feed_id_from_url(feed_url)
    if not feed_id:
        return ""
    try:
        with get_meta_connection() as conn:
            row = conn.execute("SELECT content_selector FROM scraped_feeds WHERE id = ?", (feed_id,)).fetchone()
    except Exception:  # noqa: BLE001 — table may not exist; fall back to readability
        return ""
    return str(row[0] or "").strip() if row else ""


def _scraped_entry_title(feed_url: str, entry_id: str) -> str:
    """The title of a scraped feed's entry as the scraper stored it, or "" when it has none."""
    try:
        with get_reader() as reader:
            entry = reader.get_entry((feed_url, entry_id), None)
        return str(getattr(entry, "title", "") or "").strip()
    except Exception:  # noqa: BLE001 — a missing title just falls back to the page's
        return ""


def _extract_selected_region(url: str, selector: str, capture: dict, ignore_cooldown: bool = False) -> tuple[str, str] | None:
    """Fetch *url* and return ``(title, html)`` of the region *selector* names (see scraper_service.select_region), or None."""
    from bs4 import BeautifulSoup

    result = page_fetcher.fetch(
        url, timeout=12.0, refusal_statuses=_READABILITY_REFUSAL_STATUSES, max_tier="flaresolverr", ignore_cooldown=ignore_cooldown
    )
    capture["raw_html"] = result.html
    soup = BeautifulSoup(result.html, "html.parser")
    try:
        region = scraper_service.select_region(soup, selector)
    except Exception:  # noqa: BLE001 — a bad selector falls back to readability
        return None
    if region is None:
        return None
    title = soup.title.get_text(strip=True) if soup.title else ""
    scraper_service.absolutize_links(region, url)
    return title, sanitize_readability_html(_append_site_embeds(str(region), url, result.html, prepend=True))


def _refresh_captured_article_for_current_user(
    feed_url: str,
    entry_id: str,
    mode: str = "readability",
    bump_received: bool | None = None,
    date_choice: str | None = None,
    *,
    ignore_cooldown: bool = False,
    only_if_richer: bool = False,
) -> dict:
    """Re-fetch a Lectio capture that lives on a real feed (post auto-filing),
    with the current tenancy's reader/meta DB.

    *mode* ``"full"`` captures the whole page body instead of readability-
    extracting it — the escape hatch for pages readability mangles. See
    extract_full_page_article.

    *mode* ``"archive"`` goes straight to the Wayback snapshot instead of the
    live page. The automatic archive fallback below only triggers when the live
    fetch is *refused* (parked page, 404), which leaves out the case that sent
    users to archive.org by hand: a publisher serving a page that passes every
    guard but is no longer the article — rewritten, truncated, or paywalled.

    Escalates the live fetch through services/page_fetch.py's full ladder,
    FlareSolverr rung included (unlike a plain reader-view render, this is
    never on a request's hot path — always behind run_in_threadpool or a
    background thread). *ignore_cooldown*, when True, skips the page ladder's
    own host cooldown — pass it for a user-initiated re-fetch (a person
    clicking "Refetch content" is not the polite background traffic the
    cooldown exists to pace); leave it False for the auto-refetch-on-keep
    thread and the batch worker, which already have their own host-cooldown
    mechanisms and should stay well-behaved.

    *bump_received* and *date_choice* are forwarded as-is to
    refresh_captured_article — see there.

    *only_if_richer* is for the automatic callers (auto-refetch on keep, ingest full-content fetch): the stored body is kept unless
    the fetch is richer, since "thin" can just mean a short post and readability can lock onto a site blurb."""
    from_archive: str | None = None
    if mode == CAPTURE_MODE_ARCHIVE:
        from_archive = wayback_snapshot_url(_entry_source_url(feed_url, entry_id) or entry_id)
        if not from_archive:
            return {"ok": False, "error": "The Internet Archive has no snapshot of this page."}
    # A site whose content IS its images (basslessons.be sheet-music scans) opts
    # into full-page: readability keeps the one image it scores highest and drops
    # the rest, which for a 6-page transcription loses 5 of them. An explicit
    # archive re-fetch still wins — the user asked for the snapshot.
    if mode != CAPTURE_MODE_ARCHIVE and site_content_plugins.prefers_full_page(_entry_source_url(feed_url, entry_id) or ""):
        mode = CAPTURE_MODE_FULL
    _base = fetch_full_page_article if mode == CAPTURE_MODE_FULL else fetch_readability_article
    # Keep the page we already fetched, so a date can be mined from it without a
    # second request. readability strips head metadata, which is where the date is.
    _capture: dict = {}

    _scraped_selector = _scraped_feed_content_selector(feed_url) if mode != CAPTURE_MODE_ARCHIVE else ""

    def extract(url: str):
        # A scraped feed with a content selector names the article region itself; readability on its pages keeps the site
        # chrome (texasbluesalley: nav, footer, 36 KB around a 1 KB lesson description).
        if _scraped_selector:
            region = _extract_selected_region(url, _scraped_selector, _capture, ignore_cooldown)
            if region:
                # The page's <title> is the site's name (or "Site - Post"); the scraper already gave the entry its real title.
                return _scraped_entry_title(feed_url, entry_id) or region[0], region[1]
        # An explicit archive re-fetch ignores the URL the caller would have
        # used: the snapshot IS the target.
        target = from_archive or url
        try:
            return _base(target, capture=_capture, max_tier="flaresolverr", ignore_cooldown=ignore_cooldown)
        except TypeError:
            return _base(target)  # a caller-supplied extractor without the kwarg

    reader = get_reader()
    with get_meta_connection() as conn:
        result = saved_articles_service.refresh_captured_article(
            reader,
            conn,
            feed_url,
            entry_id,
            extract=extract,
            enqueue_archive=starred_archive_service.enqueue_archive,
            is_boilerplate_extraction=starred_archive_service.extraction_matches_sibling,
            is_richer=_autofetch_is_richer if only_if_richer else None,
            bump_received=bump_received,
            date_choice=date_choice,
        )
    if from_archive and result.get("ok"):
        result["from_archive"] = from_archive
    # The live page was refused as a different article (parked page, section
    # index). Ask the archive for the real one before giving up — the guard
    # protects the stored copy, but on its own it leaves the user stuck.
    if not result.get("ok") and (result.get("mismatch") or result.get("dead")):
        snapshot = wayback_snapshot_url(result.get("source_url") or entry_id)
        if snapshot:
            LOGGER.info("[re-fetch] falling back to the archive for %s", entry_id)
            _capture.clear()

            def _from_archive(_url: str):
                return _base(snapshot, capture=_capture)

            with get_meta_connection() as conn:
                archived_result = saved_articles_service.refresh_captured_article(
                    reader,
                    conn,
                    feed_url,
                    entry_id,
                    extract=_from_archive,
                    enqueue_archive=starred_archive_service.enqueue_archive,
                    is_boilerplate_extraction=starred_archive_service.extraction_matches_sibling,
                    is_richer=_autofetch_is_richer if only_if_richer else None,
                    bump_received=bump_received,
                    date_choice=date_choice,
                )
            if archived_result.get("ok"):
                archived_result["from_archive"] = snapshot
                result = archived_result

    if result.get("ok"):
        # The body just changed, so a lead image derived from the OLD body is
        # stale — and a stale one is worse than none, because the first body
        # image is HOISTED OUT of the article to become the hero. A treblezine
        # post re-fetched from the archive ended up with a dead 403 hero (the
        # publisher's original URL, cached earlier) while the working archived
        # image was removed from the body as "already the lead". The image
        # simply vanished. Clearing it here makes the next render derive one
        # from what the article now actually contains.
        try:
            old_lead = lead_image_service.clear_entry_lead_image_cache(feed_url, entry_id)
            if old_lead:
                # Clearing is right when the new body can supply a replacement, but when it
                # holds no image at all the next render would derive nothing and the post
                # would lose a perfectly good thumbnail. Keep the old one in that case.
                entry = reader.get_entry((feed_url, entry_id), None)
                body = ""
                if entry is not None:
                    body = entry_content.entry_body(entry)
                if not lead_image_service._extract_first_image_url_from_html(body, str(getattr(entry, "link", "") or feed_url)):
                    lead_image_service.store_entry_lead_image(feed_url, entry_id, old_lead)
        except Exception:  # noqa: BLE001 — never fail a good re-fetch over this
            LOGGER.debug("lead-image invalidation failed for %s", entry_id, exc_info=True)
        try:
            result["dated"] = _apply_mined_publish_date(feed_url, entry_id, _capture.get("raw_html"))
        except Exception:  # noqa: BLE001 — a date is a bonus, never a failure
            LOGGER.warning("re-fetch date mining failed for %s", entry_id, exc_info=True)
    return result


def _apply_mined_publish_date(feed_url: str, entry_id: str, raw_html: str | None) -> str | None:
    """Set a publish date mined from the page, but ONLY when we have none.

    Deliberately narrow. Re-fetch used to move `published` and destroyed 105 real
    dates before that was caught, so this never overwrites a date the entry
    already has, and never touches an entry whose date the user pinned by hand.
    It exists for the entries still sitting on a missing-date sentinel because
    their importer had no date to give — a re-fetch is a free chance to learn one.

    Three sources, worst last (see services/publish_date):
      1. the page's own metadata,
      2. a date the page prints for humans but never marks up — hanselman.com
         ships `<span class="blogMetaDate">` and nothing machine-readable,
      3. the site's own index, for sites that publish dates nowhere near the
         article — what-if.xkcd.com articles carry no date at all.
    """
    if not raw_html:
        # Sources 2 and 3 still need trying: a site index does not need the page,
        # and this is reached when a fetch produced no HTML to mine.
        raw_html = None
    with get_meta_connection() as conn:
        if conn.execute(
            "SELECT 1 FROM entry_date_overrides WHERE feed_url = ? AND entry_id = ?",
            (feed_url, entry_id),
        ).fetchone():
            return None  # an explicit correction outranks anything inferred
    with get_reader() as reader:
        db = reader._storage.get_db()
        row = db.execute("SELECT published FROM entries WHERE feed = ? AND id = ?", (feed_url, entry_id)).fetchone()
        current_raw = row[0] if row else None
        # "We don't know" means absent OR a sentinel. This used to test only for
        # the 1970 prefix, which silently excluded the 129 entries whose importer
        # wrote year 0001 instead — the same class of value, a different spelling.
        # real_published_date is the single place that judgement lives.
        if current_raw is not None:
            parsed = current_raw
            if isinstance(parsed, str):
                try:
                    parsed = datetime.fromisoformat(parsed)
                except ValueError:
                    parsed = None
            if parsed is not None and real_published_date(parsed) is not None:
                return None  # a real date already; never overwrite it
        source = "metadata"
        mined = mine_publish_date(raw_html)
        if mined is None:
            mined, source = publish_date_service.resolve(raw_html, entry_id)
        if mined is None:
            return None
        stored = mined.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
        db.execute("UPDATE entries SET published = ? WHERE feed = ? AND id = ?", (stored, feed_url, entry_id))
        db.commit()
    invalidate_unread_counts_cache()
    LOGGER.info("[re-fetch] learned publish date %s (%s) for %s", stored, source, entry_id)
    return stored


@app.api_route("/api/save", methods=["GET", "POST"])
async def api_save_article(request: Request):
    """Token-authenticated save for mobile share sheets / iOS Shortcuts.

    Auth mirrors the Fever/GReader model: ``username`` + the per-user API
    token (query or form params), resolved to a user and bound to the tenancy
    context explicitly — this path is session-exempt. The blocking fetch runs
    in the threadpool so it can't stall the event loop."""
    params = dict(request.query_params)
    if request.method == "POST":
        try:
            # Form values can be str or UploadFile (multipart file fields); every
            # field this route reads (url/username/token) is a plain string, so
            # a file posted under one of those names is dropped rather than
            # handed to code that expects str.
            form = await request.form()
            params.update({k: v for k, v in form.items() if isinstance(v, str)})
        except Exception:  # noqa: BLE001
            pass
    url = params.get("url", "")
    uid = user_store.verify_api_token(params.get("username", ""), params.get("token", ""))
    if not uid:
        return JSONResponse({"ok": False, "error": "Authentication failed — check your API token."}, status_code=401)

    def _do_save() -> dict:
        with tenancy.user_context(uid):
            return _save_article_for_current_user(url)

    result = await run_in_threadpool(_do_save)
    return JSONResponse(result, status_code=200 if result["ok"] else 400)


def _unwrap_lectio_reading_url(url: str, request: Request) -> tuple[str, str] | None:
    """If *url* is a reading view on THIS Lectio instance, return the
    (feed_url, entry_id) it wraps, else None.

    The browser extension captures whatever tab it's on — used from inside
    Lectio, that's Lectio's own UI page, and saving that verbatim is never
    what the user means. Unwrapping turns "save this tab" into "save the
    article I'm reading here."."""
    try:
        parsed = urlparse(url)
    except ValueError:
        return None
    if parsed.scheme not in {"http", "https"}:
        return None
    own_hosts = {request.url.netloc.lower()}
    if LECTIO_PUBLIC_URL:
        own_hosts.add(urlparse(LECTIO_PUBLIC_URL).netloc.lower())
    if parsed.netloc.lower() not in own_hosts:
        return None
    qs = parse_qs(parsed.query)
    feed_url = (qs.get("feed_url") or [""])[0]
    entry_id = (qs.get("entry_id") or [""])[0]
    if feed_url and entry_id:
        return feed_url, entry_id
    return None


def _star_entry_for_current_user(feed_url: str, entry_id: str) -> dict:
    """Star an existing entry (the Lectio-native "save for later"), mirroring
    the saved-articles result shape. Deliberately no on-star destination
    fan-out, matching direct saves."""
    reader = get_reader()
    entry = reader.get_entry((feed_url, entry_id), None)
    if entry is None:
        return {
            "ok": False,
            "error": "Entry not found.",
            "duplicate": False,
            "extracted": False,
            "feed_url": feed_url,
            "entry_id": entry_id,
            "title": None,
        }
    with get_meta_connection() as conn:
        cur = conn.execute(
            "INSERT OR IGNORE INTO saved_entries (feed_url, entry_id) VALUES (?, ?)",
            (feed_url, entry_id),
        )
        conn.commit()
        newly = cur.rowcount > 0
    try:
        starred_archive_service.enqueue_archive(feed_url, entry_id)
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("extension-save: archive enqueue failed for %s/%s: %s", feed_url, entry_id, exc)
    return {
        "ok": True,
        "error": None,
        "duplicate": not newly,
        "extracted": False,
        "feed_url": feed_url,
        "entry_id": entry_id,
        "title": entry.title or entry_id,
        "starred_existing": True,
    }


def _apply_canonical_entry_link(feed_url: str, entry_id: str, old_link: str, final_url: str) -> bool:
    """Rewrite a redirector entry link (the title's href) to the canonical URL
    the redirect resolved to. Called by the starred-archive worker right after
    it fetched the source page (so canonicalization costs no extra request),
    inside the capture's tenancy context. An override row lets the refresh
    service re-pin the rewrite when the feed re-ingests the proxy link."""
    if not link_canonical.is_redirector_link(old_link) or link_canonical.is_redirector_link(final_url):
        return False
    parsed = urlparse(final_url)
    if parsed.scheme not in {"http", "https"} or not parsed.netloc or final_url == old_link:
        return False
    try:
        with get_meta_connection() as conn:
            conn.execute(
                "INSERT OR REPLACE INTO entry_link_overrides (feed_url, entry_id, link) VALUES (?, ?, ?)",
                (feed_url, entry_id, final_url),
            )
            conn.commit()
        with get_reader() as reader:
            db = reader._storage.get_db()
            db.execute(
                "UPDATE entries SET link = ? WHERE feed = ? AND id = ?",
                (final_url, feed_url, entry_id),
            )
            db.commit()
        LOGGER.info("[canonical-link] %s -> %s", old_link, final_url)
        return True
    except Exception as exc:  # noqa: BLE001
        LOGGER.warning("[canonical-link] rewrite failed for %s/%s: %s", feed_url, entry_id, exc)
        return False


_BOOKMARKLET_MAX_HTML_CHARS = 6_500_000  # mirrors the Readit extension's own cap
_BOOKMARKLET_CORS_HEADERS = {
    # Token auth lives in the JSON body (no cookies), so a wildcard origin is
    # safe — this is what lets the Readit browser extension (whose
    # host_permissions don't cover this instance) call us under normal CORS.
    "Access-Control-Allow-Origin": "*",
    "Access-Control-Allow-Methods": "POST, OPTIONS",
    "Access-Control-Allow-Headers": "Content-Type",
    "Access-Control-Max-Age": "86400",
}


@app.api_route("/api/bookmarklet/save", methods=["POST", "OPTIONS"])
async def api_bookmarklet_save(request: Request):
    """Readit-extension-compatible save protocol: Lectio as the Backend.

    Accepts the same payload the wereadit.com bookmarklet/extension sends —
    ``{token, url, html, title}`` — with ``token`` = the user's Lectio API
    token. When ``html`` is present it's the rendered DOM captured by the
    user's authenticated browser, so paywalled/bot-walled pages arrive with
    full text and no server fetch happens; without it we fall back to the
    normal server-side fetch+extract. Response mirrors the shapes the
    extension reads (`ok` on success, `detail` on failure)."""
    if request.method == "OPTIONS":
        return Response(status_code=204, headers=_BOOKMARKLET_CORS_HEADERS)
    try:
        body = await request.json()
        if not isinstance(body, dict):
            raise ValueError
    except Exception:  # noqa: BLE001
        return JSONResponse({"ok": False, "detail": "Invalid JSON body."}, status_code=400, headers=_BOOKMARKLET_CORS_HEADERS)
    token = str(body.get("token") or "")
    url = str(body.get("url") or "")
    page_html = body.get("html")
    page_title = str(body.get("title") or "").strip()
    uid = user_store.user_for_api_token(token)
    if not uid:
        return JSONResponse(
            {"ok": False, "detail": "Invalid token — paste your Lectio API token (Settings → Account)."},
            status_code=401,
            headers=_BOOKMARKLET_CORS_HEADERS,
        )
    if not (isinstance(page_html, str) and page_html.strip()):
        page_html = None
    elif _MARKDOWN_URL_RE.search(urlparse(url).path):
        # A raw .md/.md.txt page captures as the browser's <pre> wrapper —
        # drop the capture and let the server fetch + Markdown-convert.
        page_html = None
    elif len(page_html) > _BOOKMARKLET_MAX_HTML_CHARS:
        page_html = page_html[:_BOOKMARKLET_MAX_HTML_CHARS]

    def _extract_from_capture(u: str) -> tuple[str, str]:
        assert page_html is not None  # only wired in as `extract` below when it is
        title, article_html = extract_readability_article(page_html, u)
        if page_title and (not title or title == u):
            title = page_title
        return title, article_html

    # A capture from inside Lectio itself means "save the article I'm reading",
    # not Lectio's UI page: star the wrapped entry (the native save-for-later).
    wrapped = _unwrap_lectio_reading_url(url, request)

    def _do_op() -> dict:
        if wrapped is not None:
            result = _star_entry_for_current_user(*wrapped)
            if result["ok"] or not saved_articles_service.normalize_article_url(wrapped[1]):
                return result
            # Entry aged out of its feed but its id is an article URL — save
            # that instead (server fetch; the captured DOM is Lectio chrome).
            return _save_article_for_current_user(wrapped[1], None)
        extract = _extract_from_capture if page_html is not None else None
        # A captured-DOM save of an existing article is a deliberate
        # re-capture (e.g. cleaned up in-browser first): refresh the content.
        return _save_article_for_current_user(url, extract, refresh_content=page_html is not None)

    def _do_save() -> dict:
        with tenancy.user_context(uid):
            return _do_op()

    result = await run_in_threadpool(_do_save)
    if not result.get("ok") and result.get("error"):
        result["detail"] = result["error"]  # the extension surfaces `detail`
    return JSONResponse(result, status_code=200 if result["ok"] else 400, headers=_BOOKMARKLET_CORS_HEADERS)


def parse_manual_tag_edit_tokens(raw_value: str | None) -> tuple[list[str], set[str]]:
    """Split a bulk tag-edit input into (add tokens, remove-normalized-set).

    Same ``+/-tag`` convention already used by the rule editor's tag_filter
    spec (``parse_tag_filter_spec``): a leading ``-`` marks removal, a bare or
    ``+``-prefixed token marks addition. Space/comma-separated, matching the
    existing single-tag input's own tokenizing (``parse_manual_hashtags``),
    not the filter spec's comma-only split — multi-word tags aren't a manual-
    tag concern the way they are for a typed filter phrase.
    """
    add_tokens: list[str] = []
    remove_set: set[str] = set()
    seen_add: set[str] = set()
    if not raw_value:
        return add_tokens, remove_set
    for token in re.split(r"[\s,]+", raw_value):
        if not token:
            continue
        if token[0] == "-":
            normalized = normalize_tag_value(token[1:])
            if normalized:
                remove_set.add(normalized)
        else:
            token = token[1:] if token[0] == "+" else token
            normalized = normalize_tag_value(token)
            if normalized and normalized not in seen_add:
                seen_add.add(normalized)
                add_tokens.append(normalized)
    return add_tokens, remove_set


def apply_manual_tag_edits(existing_tags: list[str], add_tokens: list[str], remove_set: set[str]) -> list[str]:
    """Existing tags with ``remove_set`` dropped and ``add_tokens`` appended
    (deduped, capped at ``MAX_MANUAL_TAGS``). A tag both added and removed in
    the same edit is removed — the leading ``-`` was the more specific,
    deliberate keystroke."""
    kept = [t for t in existing_tags if t not in remove_set]
    for tok in add_tokens:
        if len(kept) >= MAX_MANUAL_TAGS:
            break
        if tok in remove_set or tok in kept:
            continue
        kept.append(tok)
    return kept


def _scope_starred_keys(folder_id: int | None, list_feed_url: str | None, tag: str | None) -> list[tuple[str, str]]:
    """The starred entries inside a drilled-down view: feed and/or tag.

    Scoped exactly as the user drilled — "a single feed with stars I don't need"
    means feed AND tag together, not either alone. Stars only: a tagged-but-
    unstarred entry has no star to remove, and unstarring is not the way to drop a
    tag (that is Delete tag everywhere).
    """
    normalized_tag = normalize_tag_value(tag)
    with get_meta_connection() as conn:
        if list_feed_url:
            feeds = {list_feed_url}
        elif folder_id is not None:
            feeds = set(get_folder_feed_urls(conn, int(folder_id)))
        else:
            feeds = set(get_all_reader_feed_urls())
        # Disabled feeds keep their stars visible, so they keep them removable too.
        starred = {(str(f), str(e)) for f, e in conn.execute("SELECT feed_url, entry_id FROM saved_entries") if str(f) in feeds}
    if normalized_tag:
        starred &= get_entry_keys_for_manual_tag(feeds, normalized_tag)
    return sorted(starred)


@overload
def _refetch_job_state(create: Literal[True]) -> dict: ...
@overload
def _refetch_job_state(create: bool = False) -> dict | None: ...
def _refetch_job_state(create: bool = False) -> dict | None:
    with _refetch_jobs_lock:
        job = _refetch_jobs.get("job")
        if job is None and create:
            job = {"running": False}
            _refetch_jobs["job"] = job
        return job


def _scope_refetchable(folder_id: int | None, list_feed_url: str | None) -> list[tuple[str, str, str]]:
    """(feed_url, entry_id, link) for kept entries in scope with a usable link.

    Kept only — the same rule the single-article button uses, because an unkept
    feed entry is rewritten by the next refresh anyway.
    """
    with get_meta_connection() as conn:
        if list_feed_url:
            feeds = {list_feed_url}
        elif folder_id is not None:
            feeds = set(get_folder_feed_urls(conn, int(folder_id)))
        else:
            feeds = set(get_all_reader_feed_urls())
        starred = {(str(f), str(e)) for f, e in conn.execute("SELECT feed_url, entry_id FROM saved_entries") if str(f) in feeds}
    kept = starred | {k for k in get_tagged_entry_keys(feeds) if k[0] in feeds}
    rows: list[tuple[str, str, str]] = []
    with get_reader() as reader:
        for f, e in sorted(kept):
            entry = reader.get_entry((f, e), None)
            if entry is None:
                continue
            link = str(getattr(entry, "link", "") or "") or e
            if link.startswith(("http://", "https://")):
                rows.append((f, e, link))
    return refetch_batch.interleave_by_host(rows)


def _refetch_scope_label(folder_id: int | None, list_feed_url: str | None) -> str:
    """Human name for a scope, so the status pill says what is running."""
    if list_feed_url:
        with get_reader() as reader:
            feed = reader.get_feed(list_feed_url, None)
        return str(getattr(feed, "user_title", None) or getattr(feed, "title", None) or list_feed_url)
    if folder_id is not None:
        with get_meta_connection() as conn:
            row = conn.execute("SELECT name FROM folders WHERE id = ?", (int(folder_id),)).fetchone()
        return str(row[0]) if row else f"folder {folder_id}"
    return "everything"


def _refetch_status_payload(job: dict | None) -> dict:
    if job is None:
        return {"ok": True, "running": False, "idle": True, "queue": [], "history": []}
    out = {k: v for k, v in job.items() if k not in ("cancel", "cancel_all", "queue", "history")}
    out["ok"] = True
    out["queue"] = [{"label": q["label"], "count": q["count"], "estimate_seconds": q["estimate_seconds"]} for q in (job.get("queue") or [])]
    out["history"] = list(job.get("history") or [])[-5:]
    return out


def _refetch_begin(job: dict, label: str, rows: list, estimate: int, date_choice: str | None = None) -> None:
    job.update(
        {
            "running": True,
            "cancel": False,
            "done": 0,
            "total": len(rows),
            "ok": 0,
            "archive": 0,
            "refused": 0,
            "dead": 0,
            "failed": 0,
            "skipped": 0,
            "scope": label,
            "started_at": time.time(),
            "finished_at": None,
            "estimate_seconds": estimate,
            "date_choice": date_choice,
        }
    )
    job.setdefault("queue", [])
    job.setdefault("history", [])


def _refetch_worker(rows: list[tuple[str, str, str]], job: dict) -> None:
    """Run the current batch, then drain the queue one scope at a time."""
    while True:
        _run_refetch_batch(rows, job)
        job["history"].append(
            {
                "scope": job.get("scope"),
                "done": job.get("done"),
                "total": job.get("total"),
                "ok": job.get("ok"),
                "archive": job.get("archive"),
                "refused": job.get("refused"),
                "dead": job.get("dead"),
                "failed": job.get("failed"),
                "finished_at": job.get("finished_at"),
                "cancelled": bool(job.get("cancel")),
            }
        )
        with _refetch_jobs_lock:
            queue = job.get("queue") or []
            if job.get("cancel_all") or not queue:
                job["cancel_all"] = False
                job["running"] = False
                return
            nxt = queue.pop(0)
        # Resolved now, not when queued: an hour in a queue is long enough for what
        # is kept in the scope to have changed.
        # An explicit selection carries its own rows; there is no scope to re-resolve.
        rows = nxt["rows"] if nxt.get("rows") is not None else _scope_refetchable(nxt["folder_id"], nxt["list_feed_url"])
        if not rows:
            continue
        with _refetch_jobs_lock:
            _refetch_begin(job, nxt["label"], rows, nxt["estimate_seconds"], date_choice=nxt.get("date_choice"))


def _run_refetch_batch(rows: list[tuple[str, str, str]], job: dict) -> None:
    host_failures: dict[str, int] = {}
    host_last: dict[str, float] = {}
    # Same "now"/"original"/"pub" picker as the single-article re-fetch, applied
    # per article below. "now" deliberately overrides bump_received=False (see
    # the call below) — that default exists to stop a routine backfill from
    # dumping the whole Inbox's order onto whatever finished last, but an
    # explicit "Now" pick is exactly asking for that, not an accidental side
    # effect of one.
    date_choice = job.get("date_choice")
    try:
        for feed_u, entry_id, link in rows:
            if job.get("cancel"):
                break
            host = refetch_batch.host_of(link)
            if host_failures.get(host, 0) >= refetch_batch.HOST_FAILURE_LIMIT:
                job["skipped"] += 1
                job["done"] += 1
                continue
            wait = refetch_batch.PER_HOST_DELAY - (time.monotonic() - host_last.get(host, 0.0))
            if wait > 0:
                time.sleep(wait)
            time.sleep(refetch_batch.GLOBAL_DELAY * (0.5 + secrets.randbelow(1000) / 1000))
            host_last[host] = time.monotonic()

            try:
                # False: a bulk backfill across dozens of old articles isn't
                # "look, this one changed" the way a deliberate single re-fetch
                # is — bumping every touched saved_at at once used to dump the
                # whole Inbox's order onto whatever finished last.
                # ignore_cooldown: the ladder's 6h host cooldown is tripped by ONE page
                # exhausting every tier (a single WAF-blocked URL), and would then
                # silently skip the rest of the host while this loop counted each skip
                # as a failure. The batch has its own pacing and HOST_FAILURE_LIMIT, so
                # a host that really is refusing us still stops the run, after
                # HOST_FAILURE_LIMIT genuine attempts rather than one.
                result = _refresh_captured_article_for_current_user(
                    feed_u, entry_id, "readability", bump_received=False, date_choice=date_choice, ignore_cooldown=True
                )
            except Exception:  # noqa: BLE001 — one bad entry must not end the run
                LOGGER.warning("[refetch-batch] failed for %s", entry_id, exc_info=True)
                result = {"ok": False}
            if result.get("ok"):
                job["archive" if result.get("from_archive") else "ok"] += 1
                host_failures[host] = 0
            elif result.get("mismatch"):
                job["refused"] += 1  # stored copy deliberately left alone
                host_failures[host] = 0
            elif result.get("dead"):
                job["dead"] += 1
                host_failures[host] = 0
            else:
                job["failed"] += 1
                host_failures[host] = host_failures.get(host, 0) + 1
            job["done"] += 1
    finally:
        # Deliberately does NOT clear `running`: the worker owns the lifecycle so
        # the status stays continuously running across a queued scope. Clearing it
        # here made the pill blink out between batches.
        job["finished_at"] = time.time()
        LOGGER.info(
            "[refetch-batch] %s: ok=%d archive=%d refused=%d dead=%d failed=%d",
            job.get("scope"),
            job["ok"],
            job["archive"],
            job["refused"],
            job["dead"],
            job["failed"],
        )


# Effectively unbounded: a range action must see the whole current view to place
# the anchor. A folder's own post count is the practical bound.
_RANGE_READ_LIMIT = 1_000_000


_UNDO_MARK_READ_WINDOW = timedelta(minutes=15)


def _undo_token_problem(raw: str) -> JSONResponse | None:
    """Parse and window-check a bulk-action undo token; None when it is good.

    The token is the batch's shared timestamp echoed back by the client, so it is untrusted input.
    ``fromisoformat`` returns an AWARE datetime for anything carrying an offset ("...+01:00"), and
    subtracting that from a naive ``datetime.now()`` raises TypeError — a 500 from a crafted value. Compare
    on a common footing by reading "now" in whatever zone the token claims.

    Shared by the mark-read and mark-unread undos so their window and token handling cannot drift.
    """
    try:
        stamped = datetime.fromisoformat(raw)
    except ValueError, TypeError:
        return JSONResponse({"ok": False, "error": "Bad undo token."}, status_code=400)
    now = datetime.now(stamped.tzinfo) if stamped.tzinfo is not None else datetime.now()
    if now - stamped > _UNDO_MARK_READ_WINDOW:
        return JSONResponse({"ok": False, "error": "The undo window has passed."}, status_code=410)
    return None


def _instapaper_save_url(username: str, password: str, url: str, title: str) -> tuple[bool, str | None]:
    """POST a URL to the Instapaper Simple API. Returns (ok, error). Instapaper
    dedupes by URL on its side, so re-saving the same URL is harmless."""
    try:
        import urllib.parse
        import urllib.request

        data = urllib.parse.urlencode(
            {
                "username": username,
                "password": password,
                "url": url,
                "title": title or "",
            }
        ).encode()
        req = urllib.request.Request(
            "https://www.instapaper.com/api/add",
            data=data,
            headers={"Content-Type": "application/x-www-form-urlencoded"},
        )
        with urllib.request.urlopen(req, timeout=10) as resp:
            status = resp.status
        if status in (200, 201):
            return True, None
        return False, f"Instapaper returned {status}."
    except Exception as exc:  # noqa: BLE001
        return False, str(exc) or "Request failed."


def _quire_add_entry(token: str, project_oid: str, title: str, link: str, feed_title: str = "") -> tuple[bool, str | None]:
    """Create a Quire task for an entry. Returns (ok, error). Raises nothing —
    rate-limit and HTTP errors are mapped to (False, message)."""
    name = (title or link or "Untitled").strip()
    desc_parts = [p for p in (link, feed_title) if p]
    description = "\n".join(desc_parts)
    try:
        quire_service.create_task(token, project_oid, name, description)
        return True, None
    except quire_service.QuireRateLimited as exc:
        return False, str(exc)
    except Exception as exc:  # noqa: BLE001
        return False, str(exc) or "Request failed."


@app.get("/api/unread-counts")
def api_unread_counts() -> JSONResponse:
    """Per-feed and per-folder unread counts plus the all-feeds total.

    The client reconciles sidebar badges from this (90s poll, tab refocus, and
    after bulk mark-read). Folder counts must come from the server: tree feed
    rows lazy-load on folder expand, so the client cannot derive an unexpanded
    folder's badge from its (absent) feed rows. Mirrors the home render's
    badge math: disabled feeds excluded, virtual Uncategorized folder counted,
    root total widened to every reader feed."""
    counts = get_unread_counts_by_feed()
    with get_meta_connection() as conn:
        snapshot = get_meta_structure_snapshot(conn)
        raw_folder_rows = cast(list[dict], snapshot["raw_folder_rows"])
        direct_feed_urls_by_folder = cast(dict[int, list[str]], snapshot["direct_feed_urls_by_folder"])
        all_feed_urls = cast(set[str], snapshot["all_feed_urls"])
        root_id = cast(int, snapshot["root_id"])
        disabled_feed_urls = get_disabled_feed_urls(conn)
    active_counts = {u: c for u, c in counts.items() if u not in disabled_feed_urls}
    folder_counts = get_unread_counts_by_folder(raw_folder_rows, active_counts, direct_feed_urls_by_folder)
    _uncat_display_urls = get_all_reader_feed_urls() - all_feed_urls - {saved_articles_service.SAVED_FEED_URL}
    uncategorized_unread = sum(active_counts.get(u, 0) for u in _uncat_display_urls)
    folder_counts[UNCATEGORIZED_FOLDER_ID] = uncategorized_unread
    folder_counts[root_id] = folder_counts.get(root_id, 0) + uncategorized_unread
    return JSONResponse(
        {
            "feeds": counts,
            "folders": folder_counts,
            "total": folder_counts.get(root_id, 0),
        }
    )


# Formats we re-encode when downscaling /api/img cache entries. Anything else
# (SVG, animated GIF/APNG, unknown) is stored byte-for-byte so we never flatten
# an animation or corrupt a vector image.
_IMG_DOWNSCALE_FORMATS = frozenset({"JPEG", "PNG", "WEBP"})

# Default freshness hint for cache hits (the upstream Cache-Control is gone by then).
_IMG_CACHE_CONTROL = "public, max-age=86400"

# Safety caps for proxied/cached images — the bytes are untrusted remote input.
#  - Byte cap: an oversized response is served through but NOT decoded or cached,
#    so a single huge image can't bloat the cache DB or be re-encoded in memory.
#  - Pixel cap: img.resize() materializes the *full source* bitmap (a 12000x8000
#    image is ~275 MB of RGB), so we skip downscaling above this and store the
#    original bytes instead — the browser decodes them, not our worker. This
#    closes a decompression-bomb / memory-DoS vector that Pillow's default
#    MAX_IMAGE_PIXELS (only trips at ~2x ~89 Mpx) leaves open.
_IMG_CACHE_MAX_BYTES = 16 * 1024 * 1024  # 16 MB
_IMG_MAX_DECODE_PIXELS = 40_000_000  # 40 megapixels
# Ceilings for the lossless-WebP path in _maybe_shrink_oversized_image. 2 MP
# keeps the lossless encoder inside a few hundred ms; 4096 colours is well below
# what painted or photographic art carries and comfortably above logos, pixel
# art and flat-colour diagrams.
_IMG_LOSSLESS_MAX_PIXELS = 2_000_000
_IMG_LOSSLESS_MAX_COLORS = 4096


def _maybe_shrink_oversized_image(raw: bytes, content_type: str, budget: int) -> tuple[bytes, str]:
    """Re-encode an image that is under the dimension cap but still enormous.

    The dimension cap alone is not a size cap. A 3840x2160 RGBA **PNG** is
    already at LECTIO_IMG_CACHE_MAX_DIM, so ``_maybe_downscale_image`` returns it
    untouched — at 11.6 MB, which is then served in full on every article view.
    Measured on a DeviantArt deviation: exactly that, and it read as "the image
    loads slowly every time" because it does. PNG is simply the wrong container
    for a photographic or painted 4K image; nothing about the pixels needs it.

    Lossless WebP is used only for small, few-colour images — logos, pixel art,
    diagrams, screenshots — where lossy compression visibly damages hard edges
    and text, and where the encoder is cheap enough to afford. Everything else
    gets lossy WebP, on the reasoning that a large full-colour image is
    photographic or painted, which is the case lossy handles best.

    **Both gates are load-bearing, and both were measured** on the 3840x2160
    deviation above. Trying lossless first and falling back cost **12.3s** in the
    request path; lossless alone on that image is 6s and still 7.9 MB. Its colour
    count is 42,082 — low enough that a naive "few colours means line art" test
    classifies painted artwork as line art and takes exactly that slow path. So
    pixels are checked first (cheap, and it is the pixel count that makes the
    encoder slow), and the colour scan only runs on images small enough for the
    answer to be both fast and true. Lossy q85/method=2 on that image: **0.25s,
    11.6 MB -> 0.17 MB.**

    Returns the original bytes and content-type unchanged on any failure, if the
    image is already within budget, or if it is animated (re-encoding would
    flatten it).
    """
    if budget <= 0 or len(raw) <= budget:
        return raw, content_type
    try:
        img = _PILImage.open(io.BytesIO(raw))
        if (img.format or "").upper() not in _IMG_DOWNSCALE_FORMATS:
            return raw, content_type
        if getattr(img, "is_animated", False):
            return raw, content_type
        w, h = img.size
        if w * h > _IMG_MAX_DECODE_PIXELS:
            return raw, content_type
        if img.mode == "P":
            img = img.convert("RGBA")

        lossless = False
        if w * h <= _IMG_LOSSLESS_MAX_PIXELS:
            # getcolors returns None once the cap is exceeded, so this bails
            # early on anything full-colour rather than counting all of them.
            lossless = img.convert("RGB").getcolors(maxcolors=_IMG_LOSSLESS_MAX_COLORS) is not None

        buf = io.BytesIO()
        if lossless:
            img.save(buf, format="WEBP", lossless=True, method=0)
        else:
            if img.mode not in ("RGB", "RGBA"):
                img = img.convert("RGBA" if "A" in img.mode else "RGB")
            img.save(buf, format="WEBP", quality=85, method=2)
        out = buf.getvalue()
        # Never hand back something bigger than what arrived — a small PNG can
        # beat WebP, and the point here is bytes, not format.
        return (out, "image/webp") if len(out) < len(raw) else (raw, content_type)
    except Exception:  # noqa: BLE001 — serving the original is always acceptable
        return raw, content_type


def _sniff_image_content_type(raw: bytes) -> str | None:
    """Identify real image bytes Pillow can decode, regardless of what
    Content-Type the server declared. Some CDNs (video.bsky.app's BunnyCDN-
    served video thumbnails, found 2026-09-11) serve a real JPEG as
    application/octet-stream — /thumb already tolerates this by decoding
    first and never checking the header; this gives /api/img the same
    tolerance instead of 422ing on an honest image. None on anything Pillow
    can't open."""
    try:
        img = _PILImage.open(io.BytesIO(raw))
        fmt = (img.format or "").upper()
    except Exception:
        return None
    return _PILImage.MIME.get(fmt) or (f"image/{fmt.lower()}" if fmt else None)


def _maybe_downscale_image(raw: bytes, max_dim: int) -> tuple[bytes, str | None]:
    """Downscale so the longest side is <= max_dim, preserving aspect ratio and
    never upscaling. Returns (bytes, content_type). The content_type is None when
    the original bytes are returned unchanged (animated/vector/unknown formats,
    already-small images, max_dim<=0, or any failure) so the caller keeps the
    upstream content-type."""
    if max_dim <= 0:
        return raw, None
    try:
        img = _PILImage.open(io.BytesIO(raw))
        fmt = (img.format or "").upper()
        # Don't touch animations (would flatten) or formats we don't re-encode.
        if fmt not in _IMG_DOWNSCALE_FORMATS or getattr(img, "is_animated", False):
            return raw, None
        w, h = img.size
        if max(w, h) <= max_dim:
            return raw, None  # already small enough; never upscale
        if w * h > _IMG_MAX_DECODE_PIXELS:
            if fmt != "JPEG":
                # Too large to resize safely (resize loads the whole source bitmap);
                # store the original bytes rather than materialize it in the worker.
                return raw, None
            # JPEG can be decoded at 1/2, 1/4 or 1/8 scale without building the full bitmap, which is what makes
            # this case safe; draft() picks the largest reduction that still covers max_dim.
            img.draft("RGB", (max_dim, max_dim))
            if img.size[0] * img.size[1] > _IMG_MAX_DECODE_PIXELS:
                return raw, None
        scale = max_dim / max(w, h)
        new_size = (max(1, round(w * scale)), max(1, round(h * scale)))
        buf = io.BytesIO()
        if fmt == "JPEG":
            if img.mode not in ("RGB", "L"):
                img = img.convert("RGB")
            img.resize(new_size, _PILImage.Resampling.LANCZOS).save(buf, format="JPEG", quality=90, optimize=True)
            return buf.getvalue(), "image/jpeg"
        if fmt == "PNG":
            # P/1-bit modes resize poorly; promote to RGBA to keep transparency.
            if img.mode in ("P", "1"):
                img = img.convert("RGBA")
            img.resize(new_size, _PILImage.Resampling.LANCZOS).save(buf, format="PNG", optimize=True)
            return buf.getvalue(), "image/png"
        # WEBP
        if img.mode == "P":
            img = img.convert("RGBA")
        img.resize(new_size, _PILImage.Resampling.LANCZOS).save(buf, format="WEBP", quality=90, method=4)
        return buf.getvalue(), "image/webp"
    except Exception:
        return raw, None


def _img_cache_get(cache_key: str) -> tuple[bytes, str] | None:
    """Return (body, content_type) for a cache hit and bump last_accessed (the
    last-accessed TTL is what keeps actively-browsed images alive). None on miss."""
    try:
        with get_img_cache_connection() as conn:
            row = conn.execute("SELECT body, content_type FROM img_cache WHERE cache_key = ?", (cache_key,)).fetchone()
            if row is None:
                return None
            conn.execute(
                "UPDATE img_cache SET last_accessed = ? WHERE cache_key = ?",
                (time.time(), cache_key),
            )
            return bytes(row["body"]), row["content_type"]
    except Exception:
        # Non-fatal (we fall back to a fetch), but log so a failing cache backend
        # — DB corruption, permissions, schema drift — isn't silently invisible.
        LOGGER.warning("[img-cache] read failed for %s; treating as miss", cache_key, exc_info=True)
        return None


def _img_cache_store(cache_key: str, body: bytes, content_type: str) -> None:
    now = time.time()
    try:
        with get_img_cache_connection() as conn:
            conn.execute(
                """
                INSERT OR REPLACE INTO img_cache
                    (cache_key, content_type, body, size, created_at, last_accessed)
                VALUES (?, ?, ?, ?, ?, ?)
                """,
                (cache_key, content_type, body, len(body), now, now),
            )
    except Exception:
        LOGGER.warning("[img-cache] store failed for %s", cache_key, exc_info=True)


# Query params that are per-request signing tokens, not image identity. Stripping them from the cache key
# lets a signed-CDN image (GitHub private-user-images JWT, wixmp/S3 ?token/X-Amz-*) stay cache-resident
# across token rotations. The full URL (with token) is still used for the actual fetch.
#
# This does NOT mean a signed image keeps loading after its URL expires, which is what this comment used to
# claim. Two preconditions have to hold: the bytes must have been fetched at least once while the token was
# still valid, and they must have survived last_accessed eviction since. Neither is guaranteed for an image
# nobody looked at in time. Measured 2026-08-18 against the live library: of 22,903 stored wixmp lead-image
# URLs, 583 had cached bytes — 2.5%. Durable protection needs pinning at ingest, not caching on demand.
_IMG_CACHE_VOLATILE_PARAMS = frozenset(
    {
        # Tapas signs its episode art as `?__token__=exp=…~acl=…` — one param
        # carrying the whole grant. Stripping it means the bytes cache once and keep
        # answering after the token expires, which is what makes a re-read of an old
        # episode free instead of a re-fetch of the page for fresh URLs.
        "__token__",
        "jwt",
        "token",
        "sig",
        "signature",
        "expires",
        "exp",
        "x-amz-algorithm",
        "x-amz-credential",
        "x-amz-date",
        "x-amz-expires",
        "x-amz-security-token",
        "x-amz-signature",
        "x-amz-signedheaders",
    }
)


def _img_cache_key_url(u: str) -> str:
    """Normalize an image URL for the cache key by dropping volatile signing params."""
    try:
        parsed = urlparse(u)
    except ValueError:
        return u
    kept = [(k, v) for (k, v) in parse_qsl(parsed.query, keep_blank_values=True) if k.lower() not in _IMG_CACHE_VOLATILE_PARAMS]
    return urlunparse(parsed._replace(query=urlencode(kept)))


# Statuses that signal a hotlink/WAF refusal of an honest image fetch — the cue to
# retry once with a same-origin Referer. Shared by api_img_proxy and thumbnail_proxy
# so the two proxies' escalation policy can't drift apart.
_HOTLINK_REFUSAL_CODES = frozenset({403, 503})


def _same_origin_referer(u: str) -> str | None:
    """Origin root (scheme://host/) of an image URL, for use as a same-origin
    Referer when a host hotlink-protects its images. None if the URL has no
    usable http(s) origin."""
    try:
        parsed = urlparse(u)
    except ValueError:
        return None
    if parsed.scheme not in ("http", "https") or not parsed.netloc:
        return None
    return urlunparse((parsed.scheme, parsed.netloc, "/", "", "", ""))


@app.get("/api/img")
async def api_img_proxy(u: str) -> Response:
    """Server-side image proxy with a content-addressed cache.

    Fetches external images on behalf of the browser so that
    Cross-Origin-Resource-Policy restrictions (same-site / same-origin) set by
    the image server do not prevent them from loading in the entry pane.
    Only http:// and https:// URLs are accepted. SSRF is prevented by
    url_guard.safe_get_async, which validates the initial URL and every redirect
    hop against private / loopback / link-local IP space.

    On a cache miss the fetched bytes are optionally downscaled (longest side ->
    LECTIO_IMG_CACHE_MAX_DIM) and stored in the global img_cache, keyed by a hash
    of the source URL. Entries are evicted by last-accessed TTL in daily
    maintenance (see _run_global_maintenance), so this also makes hotlink images
    behind short-lived signed URLs (e.g. wixmp) survive token expiry.
    """
    if urlparse(u).scheme not in ("http", "https"):
        return Response(status_code=400)
    cache_key = hashlib.sha256(_img_cache_key_url(u).encode("utf-8")).hexdigest()
    cached = _img_cache_get(cache_key)
    if cached is not None:
        body, content_type = cached
        return Response(content=body, media_type=content_type, headers={"Cache-Control": _IMG_CACHE_CONTROL})
    try:
        # follow_redirects=False so safe_get_async controls (and re-validates)
        # each hop instead of httpx silently bouncing to an internal address.
        # A WAF challenge (Cloudflare and the like) a page fetch already solved
        # via FlareSolverr for this same host leaves domain-scoped cookies
        # behind (services/page_fetch.py) — presenting them here can pass the
        # same check a plain image fetch could never clear on its own, since
        # FlareSolverr itself has no way to hand back arbitrary binary bytes.
        client_kwargs: dict[str, object] = {"timeout": 12.0}
        stored_cookies = page_fetcher.cookies_for_host(u)
        if stored_cookies:
            client_kwargs["cookies"] = stored_cookies
        async with url_guard.build_async_client(**client_kwargs) as client:
            headers = {"User-Agent": READABILITY_USER_AGENT}
            resp = await url_guard.safe_get_async(client, u, headers=headers)
            # Hotlink protection: some hosts serve a 403 (often text/html) for an
            # image fetched without a Referer, but 200 image/* once a same-origin
            # Referer is present (e.g. fabiensanglard.net's .webp files). Only
            # after an honest request is actually refused do we retry with the
            # image's own origin as Referer — never preemptively. Mirrors the
            # honest-first WAF escalation in services/lead_images.py.
            if resp.status_code in _HOTLINK_REFUSAL_CODES:
                referer = _same_origin_referer(u)
                if referer:
                    resp = await url_guard.safe_get_async(client, u, headers={**headers, "Referer": referer})
    except url_guard.UnsafeURLError:
        return Response(status_code=403)
    except Exception:
        return Response(status_code=502)
    content_type = resp.headers.get("content-type", "")
    body = resp.content
    if not content_type.startswith("image/"):
        sniffed = _sniff_image_content_type(body)
        if sniffed is None:
            return Response(status_code=422)
        content_type = sniffed
    cache_ctrl = resp.headers.get("cache-control", _IMG_CACHE_CONTROL)
    if len(body) > _IMG_CACHE_MAX_BYTES:
        # Too large to cache/re-encode; pass the original through untouched so it
        # still displays, but don't decode or store it.
        return Response(content=body, media_type=content_type, headers={"Cache-Control": cache_ctrl})
    # Both steps decode and re-encode a full bitmap, which on a 4K image is
    # hundreds of milliseconds. This is an async route, so running them inline
    # blocks the event loop — every other request on the worker waits behind one
    # large image.
    # Both settings are read HERE, on the request path, so the threadpool call
    # below is pure CPU with no DB access and no tenancy context to carry.
    _max_dim = get_img_cache_max_dim()
    _target_bytes = get_img_target_bytes()

    def _recompress(raw: bytes, ct: str) -> tuple[bytes, str]:
        downscaled, new_ct = _maybe_downscale_image(raw, _max_dim)
        if new_ct is not None:
            raw, ct = downscaled, new_ct
        # The dimension cap is not a size cap: an image already at max_dim comes
        # through untouched however many megabytes it weighs.
        return _maybe_shrink_oversized_image(raw, ct, _target_bytes)

    body, content_type = await run_in_threadpool(_recompress, body, content_type)
    _img_cache_store(cache_key, body, content_type)
    return Response(
        content=body,
        media_type=content_type,
        headers={"Cache-Control": cache_ctrl},
    )


_FAVICON_CACHE_CONTROL = "public, max-age=86400"
_FAVICON_FALLBACK_PATH = BASE_DIR / "static" / "favicon-fallback.svg"


@app.get("/api/favicon")
async def api_favicon(domain: str) -> Response:
    """Favicon resolver with a three-hop fallback chain.

    Tries in order:
      1. Google's faviconV2 service (via s2/favicons redirect).
      2. The site's own /favicon.ico.
      3. A bundled neutral SVG placeholder.

    Every outbound fetch goes through url_guard.safe_get_async with
    follow_redirects=False so SSRF is prevented on the domain parameter and all
    redirect hops. Results (including the placeholder) are cached in the shared
    img_cache DB under the key "favicon:<host>", so repeat lookups skip the
    chain entirely.

    Auth-exempt and CSRF-exempt (same as /api/img) — the browser's <img> tag
    fetches it without cookies/tokens.
    """
    # Reject clearly unsafe domain values before any DNS/network work.
    host = domain.strip().lower()
    if not host:
        return Response(status_code=400)
    # Reject anything carrying a path or an explicit scheme prefix (SSRF vector).
    # Legitimate domain params are plain hostnames like "example.com". Match the
    # scheme as "http:"/"https:" (not a bare "http" prefix, which would wrongly
    # reject real hosts such as "httpbin.org").
    if "/" in host or host.startswith(("http:", "https:")):
        return Response(status_code=400)

    cache_key = f"favicon:{host}"
    cached = _img_cache_get(cache_key)
    if cached is not None:
        body, content_type = cached
        return Response(content=body, media_type=content_type, headers={"Cache-Control": _FAVICON_CACHE_CONTROL})

    google_url = f"https://www.google.com/s2/favicons?domain={quote_plus(host)}&sz=32"
    favicon_ico_url = f"https://{host}/favicon.ico"

    async with url_guard.build_async_client(timeout=8.0) as client:
        # Hop 1: Google faviconV2 (via redirect from s2/favicons).
        try:
            resp = await url_guard.safe_get_async(client, google_url, headers={"User-Agent": READABILITY_USER_AGENT})
            ct = resp.headers.get("content-type", "")
            if resp.status_code == 200 and ct.startswith("image/"):
                body = resp.content
                _img_cache_store(cache_key, body, ct)
                return Response(content=body, media_type=ct, headers={"Cache-Control": _FAVICON_CACHE_CONTROL})
        except url_guard.UnsafeURLError, Exception:
            pass

        # Hop 2: site's own /favicon.ico.
        if url_guard.is_safe_outbound_url(favicon_ico_url):
            try:
                resp = await url_guard.safe_get_async(client, favicon_ico_url, headers={"User-Agent": READABILITY_USER_AGENT})
                ct = resp.headers.get("content-type", "")
                if resp.status_code == 200 and (ct.startswith("image/") or ct.startswith("application/octet")):
                    body = resp.content
                    ct_store = ct if ct.startswith("image/") else "image/x-icon"
                    _img_cache_store(cache_key, body, ct_store)
                    return Response(content=body, media_type=ct_store, headers={"Cache-Control": _FAVICON_CACHE_CONTROL})
            except url_guard.UnsafeURLError, Exception:
                pass

    # Hop 3: bundled neutral SVG placeholder.
    try:
        placeholder = _FAVICON_FALLBACK_PATH.read_bytes()
    except OSError:
        return Response(status_code=502)
    _img_cache_store(cache_key, placeholder, "image/svg+xml")
    return Response(content=placeholder, media_type="image/svg+xml", headers={"Cache-Control": _FAVICON_CACHE_CONTROL})


# ---------------------------------------------------------------------------
# Background-thread tenancy helpers shared by several route modules (moved
# routes/compat_fever.py, routes/compat_greader.py, and the integrations_*
# modules all import _run_in_user_context back from here).
# ---------------------------------------------------------------------------


def _run_in_user_context(uid: str, fn, *args, **kwargs) -> None:
    """Run ``fn`` in a background thread under tenancy user ``uid``.

    Manually-created threads do not inherit contextvars, so a request that
    captures its user and hands work to a daemon thread must re-bind the context
    there or the work raises TenancyUnboundError."""
    with tenancy.user_context(uid):
        fn(*args, **kwargs)


def _enhance_feeds_background(feed_urls: list[str]) -> None:
    """Run lead-image + YouTube-duration enhancement for ``feed_urls``, skipping
    feeds another enhancement run is already handling so concurrent manual /
    scheduled refreshes don't duplicate the network work."""
    # Ingest page work (full-content fetch, page topics) doesn't depend on enhancement, so it starts first, for every refreshed
    # feed. Started after, it waited out the whole batch's lead-image pass (minutes on a large refresh) and never ran at all for a
    # feed whose enhancement was skipped as already in flight.
    _spawn_full_content_drain(list(feed_urls))
    with _enhancement_inflight_lock:
        todo = [u for u in feed_urls if u not in _enhancement_inflight_feeds]
        _enhancement_inflight_feeds.update(todo)
    if not todo:
        return
    try:
        feed_refresh_service.enhance_feeds(todo)
    finally:
        with _enhancement_inflight_lock:
            _enhancement_inflight_feeds.difference_update(todo)


def _spawn_feed_enhancement(feed_urls: Iterable[str]) -> None:
    """Spawn the lead-image / duration enhancement on a daemon thread bound to
    the current tenancy user, so a refresh request can return immediately."""
    urls = list(feed_urls)
    if not urls:
        return
    threading.Thread(
        target=_run_in_user_context,
        args=(tenancy.current_user_id(), _enhance_feeds_background, urls),
        daemon=True,
    ).start()


# ---------------------------------------------------------------------------
# services/inoreader_import.py (Plan.md's main.py/index.html breakup, Stage E):
# the scheduled-refresh loop above calls _inoreader_drip_step() directly, not
# through the route. Imported here for the same reason as the route modules
# below — the module does `from main import get_inoreader_token, ...`, which
# only resolves once those names already exist in this module's namespace.
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# services/automation_rules.py (Plan.md's main.py/index.html breakup, Step 2,
# Stages B-E, now closed out): toggle_feed_tag_filter, _flush_email_batch_for_rule,
# _run_on_star_destinations, the scheduled-refresh loop and WebSub fan-out
# above, the bg-refresh thread, 3 refresh routes, and the manual Run Now /
# dry-run routes all still call one of these directly, not through a route of
# their own, so they need the same late-import treatment as
# _inoreader_drip_step above — the module does `from main import
# build_keyword_matcher, ...`, which only resolves once main.py has already
# defined those names.
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Extracted route modules (Plan.md's main.py/index.html breakup). Imported
# here rather than at the top of the file: each module does `from main
# import ...` for shared connection/setting/credential helpers, which only
# resolves once those names exist in this module's namespace.
# ---------------------------------------------------------------------------
from routes.admin import router as _admin_router  # noqa: E402, I001
from routes.compat_fever import router as _compat_fever_router  # noqa: E402
from routes.compat_greader import router as _compat_greader_router  # noqa: E402
from routes.compat_v1 import router as _compat_v1_router  # noqa: E402
from routes.entries import router as _entries_router  # noqa: E402
from routes.highlights import router as _highlights_router  # noqa: E402
from routes.home import router as _home_router  # noqa: E402
from routes.integrations_freshrss import router as _freshrss_import_router  # noqa: E402
from routes.integrations_inoreader import router as _inoreader_router  # noqa: E402
from routes.integrations_miniflux import router as _miniflux_import_router  # noqa: E402
from routes.integrations_pinterest import router as _pinterest_oauth_router  # noqa: E402
from routes.integrations_quire import router as _quire_oauth_router  # noqa: E402
from routes.integrations_reddit import router as _reddit_oauth_router  # noqa: E402
from routes.integrations_ttrss import router as _ttrss_import_router  # noqa: E402
from routes.integrations_youtube import router as _youtube_oauth_router  # noqa: E402
from routes.saved import router as _saved_router  # noqa: E402
from routes.settings import router as _settings_router  # noqa: E402
from routes.tags import router as _tags_router  # noqa: E402
from services.automation_rules import (  # noqa: E402
    _run_mark_as_read_rules_for_feed,
    _run_tag_filter_rules_for_feed,
    # The six _run_*_rules_after_refresh dispatchers and _apply_youtube_playlist_rules/
    # _entry_matches_rule have no caller left inside main.py itself now that
    # _run_automation_after_refresh has moved too -- kept importable here only for tests
    # that call main.<name> directly, and (for _apply_youtube_playlist_rules/
    # _entry_matches_rule) scripts/backfill_missed_youtube_playlist_adds.py.
    # _run_now_dedup/_run_now_pattern lost their last main.py-resident callers when
    # routes/automation.py's rules_run_now_route moved out (Stage 4 of the
    # route-by-URL-prefix split) -- same "tests call main.<name> directly" reason.
    _apply_youtube_playlist_rules,  # noqa: F401
    _entry_matches_rule,  # noqa: F401
    _get_entry_excerpt,
    _log_auto_run,
    _run_automation_after_refresh,
    _run_email_rules_after_refresh,  # noqa: F401
    _run_instapaper_rules_after_refresh,  # noqa: F401
    _run_now_dedup,  # noqa: F401
    _run_now_pattern,  # noqa: F401
    _run_quire_rules_after_refresh,  # noqa: F401
    _run_save_article_rules_after_refresh,  # noqa: F401
    _run_tag_filter,
    _run_webhook_rules_after_refresh,  # noqa: F401
    _run_youtube_playlist_rules_after_refresh,  # noqa: F401
)
from services.inoreader_import import _inoreader_drip_step  # noqa: E402

# routes.feeds (Stage 8E of the route-by-URL-prefix split) does `from main import
# (..., _run_automation_after_refresh, ...)` for /feeds/bulk's "refresh" action, so
# it must be imported after the services.automation_rules import above binds that
# name into this module's namespace -- the same ordering constraint routes.system
# hits below, moved here for the same reason once Stage 8E's bulk-action route
# started calling it directly instead of only through /refresh's own main.py-resident
# call sites.
from routes.feeds import router as _feeds_router  # noqa: E402

# routes.integrations_deviantart does `from routes.feeds import bulk_feed_action`
# (the DeviantArt watchlist auto-pause path reuses the same bulk-unsubscribe action
# Stage 8E moved into routes.feeds), so it must be imported after routes.feeds above
# -- not just after services.automation_rules, since the name it needs now lives in
# routes.feeds's own namespace, not main's.
from routes.integrations_deviantart import router as _deviantart_oauth_router  # noqa: E402

# routes.system does `from main import (..., _run_automation_after_refresh, ...)`,
# so it must be imported after the services.automation_rules import above binds
# that name into this module's namespace.
from routes.system import router as _system_router  # noqa: E402

# routes.automation does `from services.automation_rules import (_run_tag_filter,
# _run_now_dedup, _run_now_pattern)` directly rather than round-tripping them
# through main -- placed after the services.automation_rules import above so
# that module is already fully loaded (and in sys.modules) by the time
# routes.automation reaches for it, rather than routes.automation's own import
# being what first triggers services.automation_rules' module-level `from main
# import (...)` mid-way through main.py's own execution.
from routes.automation import router as _automation_router  # noqa: E402

app.include_router(_admin_router)
app.include_router(_compat_fever_router)
app.include_router(_compat_greader_router)
app.include_router(_compat_v1_router)
app.include_router(_deviantart_oauth_router)
app.include_router(_entries_router)
app.include_router(_feeds_router)
app.include_router(_quire_oauth_router)
app.include_router(_youtube_oauth_router)
app.include_router(_pinterest_oauth_router)
app.include_router(_reddit_oauth_router)
app.include_router(_inoreader_router)
app.include_router(_miniflux_import_router)
app.include_router(_freshrss_import_router)
app.include_router(_ttrss_import_router)
app.include_router(_highlights_router)
app.include_router(_home_router)
app.include_router(_saved_router)
app.include_router(_settings_router)
app.include_router(_tags_router)
app.include_router(_system_router)
app.include_router(_automation_router)


if __name__ == "__main__":
    import uvicorn

    uvicorn.run("main:app", host="127.0.0.1", port=8000, reload=True)