# dhtcrawler4 — hardened single-host deployment.
#
#   cd deploy && ../scripts/gen-secrets.sh && docker compose up -d --build
#
# Exposure (requirement R12):
#   * UDP 6881 (DHT) on all interfaces — the only public port.
#   * The web UI on 127.0.0.1:8080 — put a TLS reverse proxy in front of it.
#   * PostgreSQL has no published port and sits on an internal network.
# Every service runs non-root, read-only, with all capabilities dropped.

name: dhtcrawler4

x-dc4: &dc4
  build:
    context: ..
    dockerfile: Dockerfile
  image: dhtcrawler4:local
  read_only: true
  cap_drop: [ALL]
  security_opt: ["no-new-privileges:true"]
  pids_limit: 512
  tmpfs: ["/tmp:size=16m,mode=1777"]
  restart: unless-stopped
  environment: &dc4-env
    DC4_DATABASE__HOST: db
    DC4_DATABASE__PASSWORD_FILE: /run/secrets/db_password
    DC4_LOG__FORMAT: json
    # Metrics and health listen inside the container; only the internal backend
    # network can reach them.
    DC4_METRICS__LISTEN: "0.0.0.0:9100"

services:
  db:
    image: postgres:18.6@sha256:4ef4dbc939d61acea57712655ddb4b4ab27419c913f94cca0cd57cb3ea3c2280
    # Headroom for the roles' pools (crawl alone may hold 128).
    # Fewer, bigger checkpoints for the write-hot queue: less
    # full-page-write amplification at ~758 updates/s. Costs longer crash
    # recovery and up to ~4 GB of WAL on disk.
    command: ["postgres", "-c", "max_connections=200",
              "-c", "checkpoint_timeout=900", "-c", "max_wal_size=4GB"]
    restart: unless-stopped
    shm_size: 256m
    security_opt: ["no-new-privileges:true"]
    # The official entrypoint needs these to initialise and drop to the postgres user.
    cap_drop: [ALL]
    cap_add: [CHOWN, DAC_OVERRIDE, FOWNER, SETGID, SETUID]
    environment:
      POSTGRES_PASSWORD_FILE: /run/secrets/pg_superuser_password
      DC4_OWNER_PASSWORD_FILE: /run/secrets/dc4_owner_password
      DC4_CRAWLER_PASSWORD_FILE: /run/secrets/dc4_crawler_password
      DC4_INDEXER_PASSWORD_FILE: /run/secrets/dc4_indexer_password
      DC4_WEB_PASSWORD_FILE: /run/secrets/dc4_web_password
    secrets:
      - pg_superuser_password
      - dc4_owner_password
      - dc4_crawler_password
      - dc4_indexer_password
      - dc4_web_password
    volumes:
      - pgdata:/var/lib/postgresql
      - ./postgres/init:/docker-entrypoint-initdb.d:ro
    networks: [backend]
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres -d dc4"]
      interval: 5s
      timeout: 5s
      retries: 20
    mem_limit: 3g

  migrate:
    <<: *dc4
    command: ["migrate"]
    restart: "no"
    environment:
      <<: *dc4-env
      DC4_DATABASE__USER: dc4_owner
    secrets:
      - source: dc4_owner_password
        target: db_password
    networks: [backend]
    depends_on:
      db:
        condition: service_healthy
    mem_limit: 256m

  crawl:
    <<: *dc4
    command: ["crawl"]
    environment:
      <<: *dc4-env
      DC4_DATABASE__USER: dc4_crawler
      # 6144 fetch workers plus admission and scrapes share this pool.
      DC4_DATABASE__MAX_CONNECTIONS: "128"
    secrets:
      - source: dc4_crawler_password
        target: db_password
    ports:
      - "6881:6881/udp"   # IPv4 and IPv6. With crawl.identities = N > 1,
                          # publish "6881-688N:6881-688N/udp" and open the
                          # same range on the firewall instead.
                          # Example for N = 8 (commented: uncomment and drop
                          # the single-port line above when identities > 1):
                          # - "6881-6888:6881-6888/udp"
    volumes:
      - state:/var/lib/dhtcrawler4
    networks: [backend, egress]
    depends_on:
      migrate:
        condition: service_completed_successfully
    healthcheck:
      test: ["CMD", "/usr/local/bin/dhtcrawler4", "healthcheck", "http://127.0.0.1:9100/readyz"]
      interval: 30s
      timeout: 5s
      retries: 10
      start_period: 5m
    # Shutdown waits up to 30 s for fetches, flushes discoveries and saves DHT state.
    stop_grace_period: 60s
    # Crawl memory: in-flight metadata (1 GiB byte budget) plus routing and caches.
    mem_limit: 6g

  index:
    <<: *dc4
    command: ["index"]
    environment:
      <<: *dc4-env
      DC4_DATABASE__USER: dc4_indexer
      DC4_DATABASE__MAX_CONNECTIONS: "4"
    secrets:
      - source: dc4_indexer_password
        target: db_password
    volumes:
      - index:/var/lib/dhtcrawler4/index
    networks: [backend]
    depends_on:
      migrate:
        condition: service_completed_successfully
    healthcheck:
      test: ["CMD", "/usr/local/bin/dhtcrawler4", "healthcheck", "http://127.0.0.1:9100/readyz"]
      interval: 30s
      timeout: 5s
      retries: 10
      start_period: 2m
    mem_limit: 1g

  web:
    <<: *dc4
    command: ["web"]
    environment:
      <<: *dc4-env
      DC4_DATABASE__USER: dc4_web
      DC4_WEB__LISTEN: "0.0.0.0:8080"
      # Connections to the published port arrive from the frontend network's
      # gateway (docker-proxy), so that subnet is the trusted proxy.
      DC4_WEB__TRUSTED_PROXIES: "172.30.80.0/24"
    secrets:
      - source: dc4_web_password
        target: db_password
    ports:
      - "80:8080"
    # Tantivy readers take a short-lived lock file, so the index volume is writable here.
    volumes:
      - index:/var/lib/dhtcrawler4/index
    networks: [backend, frontend]
    depends_on:
      migrate:
        condition: service_completed_successfully
    healthcheck:
      test: ["CMD", "/usr/local/bin/dhtcrawler4", "healthcheck", "http://127.0.0.1:8080/healthz"]
      interval: 15s
      timeout: 5s
      retries: 3
    mem_limit: 512m

networks:
  backend:
    internal: true
  frontend:
    ipam:
      config:
        - subnet: 172.30.80.0/24
  egress:
    # The DHT runs over IPv4 and IPv6 (BEP 32). Requires Docker Engine 27+ (or an
    # IPv6-enabled daemon); remove this line to run IPv4-only.
    enable_ipv6: true

volumes:
  pgdata: {}
  state: {}
  index: {}

secrets:
  pg_superuser_password:
    file: ./secrets/pg_superuser_password
  dc4_owner_password:
    file: ./secrets/dc4_owner_password
  dc4_crawler_password:
    file: ./secrets/dc4_crawler_password
  dc4_indexer_password:
    file: ./secrets/dc4_indexer_password
  dc4_web_password:
    file: ./secrets/dc4_web_password