# Runs on primary, next to the sources. See IMPLEMENTATION_PLAN.md section 2.
#
# The webserver and the daemon are separate services on purpose: the daemon runs
# schedules, sensors and backfills, and the UI must stay reachable to diagnose a
# failure even when the daemon is the thing that has fallen over.

services:
  dagster-webserver:
    build: .
    container_name: datalake-webserver
    restart: unless-stopped
    env_file: [.env]
    # Dagster's run/event-log/schedule storage now lives in the shared Postgres
    # rather than its default SQLite -- see dagster.yaml for why.
    #
    # This container is the only one that holds pooled connections OPEN. The
    # webserver calls optimize_for_webserver(), which swaps each of the three
    # storage classes' default NullPool for a real pool of pool_size=1 plus
    # max_overflow. Stock max_overflow is 20, so the default behaviour is up to
    # 3 x 21 = 63 connections from this one container -- against a server-wide
    # max_connections of 100 that keycloak, banking-tracker, ai_platform and
    # litellm all draw on too. 5 caps it at 3 x 6 = 18, ample for a UI with one
    # user. The dagster_rw role additionally carries a hard CONNECTION LIMIT of
    # 40 in Postgres: this flag is the polite limit, that one cannot be bypassed
    # by any config change made here.
    #
    # Compose replaces a command wholesale rather than appending to it, so naming
    # one flag means restating the image's own CMD (-h/-p/-m) around it. This
    # container had no `command:` at all before this change.
    command:
      - dagster-webserver
      - -h
      - "0.0.0.0"
      - -p
      - "3000"
      - -w
      - /app/workspace.yaml
      - --db-pool-max-overflow
      - "5"
    # No published port on purpose. Nginx Proxy Manager joins this network and
    # reaches the UI by container name, so Dagster -- which has no authentication of
    # its own -- is not listening anywhere an unauthenticated caller can reach.
    # See infra/npm/http.conf.
    expose:
      - "3000"
    # Two networks. `default` is this project's own; `primary_reseau` is where the
    # shared Postgres lives, and the LiteLLM ingest reads its spend ledger by
    # container name. Declared here rather than attached by hand with
    # `docker network connect`, which is how it was done before: a hand-made
    # attachment does not survive `compose up`, and the one that vanished took the
    # LiteLLM source down for two days without a sound. See IMPLEMENTATION_PLAN §8a.
    networks: [default, primary_reseau]
    volumes:
      - dagster_home:/dagster_home
      - warehouse:/warehouse
      # dagster.yaml is Dagster's own instance config file. It takes no flag and is
      # auto-discovered at $DAGSTER_HOME/dagster.yaml, so it has to land inside the
      # dagster_home volume's path rather than beside the code -- hence a bind mount
      # layered over the volume mount above.
      #
      # Deliberately per-service, and deliberately NOT written into the volume
      # itself: datalake-api and datalake-mcp mount the same dagster_home volume and
      # never open a DagsterInstance, so leaving the file out of the volume is what
      # keeps this change invisible to them.
      #
      # Read-only, because nothing running in this container should be able to
      # rewrite its own storage configuration.
      - ./dagster.yaml:/dagster_home/dagster.yaml:ro
      # workspace.yaml lists the TWO code locations `-w` loads (see that file):
      # this project's own module, plus ai_automations, served over gRPC by its
      # own container on primary_reseau. Read-only for the same reason
      # dagster.yaml is -- nothing in here should be able to widen the set of
      # code locations it loads.
      - ./workspace.yaml:/app/workspace.yaml:ro
      # NPM's live log volume, READ-ONLY. A pipeline able to corrupt the reverse
      # proxy's logs is a pipeline able to take every service on the box offline.
      - ${PRIMARY_LOG_DIR:-/srv/npm/data/logs}:/logs/primary:ro
      - geoip:/geoip:ro
      # Sources added 2026-09-15, every one read-only (docs/METRICS_STREAMING_PLAN.md).
      # /var/log for the journal and auth.log. Radarr's and Sonarr's whole config
      # directories, because SQLite keeps -wal and -shm files beside the database and
      # they must be copied with it; config.xml in there holds an API key the pipeline
      # never opens. Jellyseerr's db/ directory only. The status page's data volume.
      - ${HOST_LOG_MOUNT:-/var/log}:/host/log:ro
      - ${RADARR_CONFIG_DIR:-/srv/media/Radarr/config}:/sources/radarr:ro
      - ${SONARR_CONFIG_DIR:-/srv/media/Sonarr/config}:/sources/sonarr:ro
      - ${JELLYSEERR_DB_DIR:-/srv/media/Jellyseer/config/db}:/sources/jellyseerr:ro
      - status-page-data:/sources/status-page:ro
      # mcp.ai.example.test's own access log (WS-41.E2 part C), same posture as
      # primary's NPM logs above: read-only, so mcp_api_access_raw can corrupt
      # nothing upstream of it. The default path is owned by a different repo --
      # ai-platform's own compose bind-mounts ./logs/mcp-api into the nginx
      # container in front of the MCP gateway, which on primary resolves to
      # /srv/ai-platform/deployment/logs/mcp-api. Override
      # MCP_API_LOG_DIR_HOST if that path ever moves. MCP_API_LOG_DIR in .env is
      # the matching in-container path this mounts to; see resources.NpmLogSources.
      - ${MCP_API_LOG_DIR_HOST:-/srv/ai-platform/deployment/logs/mcp-api}:/logs/mcp-api:ro
    environment:
      # Prometheus's published port, reached from this container over the LAN address.
      PROMETHEUS_URL: ${PROMETHEUS_URL:-http://198.51.100.26:9090}

  # Keycloak SSO in front of the Dagster UI.
  #
  # Dagster's webserver has no authentication of its own, so something must sit in
  # front. oauth2-proxy rather than nginx basic auth because identity belongs in one
  # place -- the same realm that holds every other account -- so access is granted and
  # revoked in Keycloak like everything else, not in an htpasswd file nobody remembers.
  #
  # It proxies rather than only authenticating: nginx sends everything here, and only
  # a request carrying a valid session for a user holding the datalake-user role is
  # forwarded to Dagster. Dagster itself is reachable from nowhere else.
  oauth2-proxy:
    image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1
    container_name: datalake-oauth2-proxy
    restart: unless-stopped
    depends_on: [dagster-webserver]
    env_file: [.env]
    environment:
      OAUTH2_PROXY_PROVIDER: keycloak-oidc
      OAUTH2_PROXY_HTTP_ADDRESS: 0.0.0.0:4180
      OAUTH2_PROXY_UPSTREAMS: http://dagster-webserver:3000
      # Trust the X-Forwarded-* headers nginx sets; without this oauth2-proxy builds
      # redirect URLs from the internal address and the login loop never terminates.
      OAUTH2_PROXY_REVERSE_PROXY: "true"
      OAUTH2_PROXY_COOKIE_SECURE: "true"
      # Straight to Keycloak instead of showing a "Sign in with..." interstitial.
      OAUTH2_PROXY_SKIP_PROVIDER_BUTTON: "true"
      # `roles` is what puts realm_access.roles in the token, which is what
      # OAUTH2_PROXY_ALLOWED_ROLES is checked against.
      OAUTH2_PROXY_SCOPE: "openid email profile roles"
      # Authorisation is the role, not the address. Any realm account can
      # authenticate; only one holding datalake-user gets through.
      OAUTH2_PROXY_EMAIL_DOMAINS: "*"
      # Keycloak advertises PKCE and oauth2-proxy will not use it unless asked.
      # It binds the authorisation code to the client that requested it, so a code
      # intercepted in the redirect cannot be exchanged by anyone else.
      OAUTH2_PROXY_CODE_CHALLENGE_METHOD: S256
      # Keep the tokens out of the session cookie.
      #
      # By default oauth2-proxy stores the access, ID and refresh tokens in the
      # cookie. Keycloak's tokens are verbose -- a service account with a single role
      # is already [count withheld] bytes -- so a real user's session runs to roughly [volume withheld]
      # against a [count withheld]-byte limit, and oauth2-proxy splits it across numbered
      # cookies with a warning. That splitting is where the login loop came from.
      #
      # Nothing downstream needs those tokens: Dagster has no authentication of its
      # own and is never handed one. The role check happens when the session is
      # created and its result is what the session carries, so authorisation is
      # unchanged. The cost is no silent token refresh -- the session simply expires
      # and asks for a login again, which for a UI is the right trade against a
      # cookie that does not survive being split.
      OAUTH2_PROXY_SESSION_COOKIE_MINIMAL: "true"
    expose:
      - "4180"

  # The MCP server: typed analytical tools over the marts, for models to call.
  #
  # It reads the *serving copy*, not the warehouse, so a query never contends with
  # the nightly build -- DuckDB allows one writing process, and a read-only open
  # fails outright under that lock rather than waiting.
  #
  # No published port, like everything else here. Nginx reaches it by container name
  # and it verifies a Keycloak-signed bearer token on every request, so being on this
  # network is not itself authorisation.
  datalake-mcp:
    build: .
    container_name: datalake-mcp
    restart: unless-stopped
    env_file: [.env]
    command: ["python", "-m", "homelab_datalake.mcp.server"]
    environment:
      MCP_TRANSPORT: streamable-http
      MCP_PORT: "3333"
      MCP_PUBLIC_URL: https://data.example.test/mcp
    expose:
      - "3333"
    volumes:
      # Read-only: this container answers questions, it does not build anything. If
      # it could write to the warehouse volume it could also take the write lock and
      # stall the nightly job.
      - warehouse:/warehouse:ro
      - exports:/exports

  # The workbench API: the catalogue, and SQL over the whole lake.
  #
  # Distinct from datalake-mcp on purpose. That one answers *typed* questions over
  # the marts and physically cannot reach anything else, because the serving copy it
  # opens contains only marts. This one reads raw, curated and marts through DuckDB
  # over the Parquet, which is a much wider surface -- so every guard it has is code
  # in homelab_datalake.api, tested with the rest of the repository, rather than
  # something the browser is trusted to enforce.
  #
  # On primary_reseau as well as the project network, because the caller is the
  # ai-platform gateway: the browser never reaches this, and neither does the Python
  # sandbox, which has no route off its own internal network and relays through that
  # same gateway.
  datalake-api:
    build: .
    container_name: datalake-api
    restart: unless-stopped
    env_file: [.env]
    command: ["python", "-m", "homelab_datalake.api.app"]
    environment:
      DATALAKE_API_PORT: "3444"
    # No published port, like everything else here. Being on this network is not
    # authorisation: every route but /health verifies a Keycloak-signed bearer token.
    expose:
      - "3444"
    networks: [default, primary_reseau]
    volumes:
      # Read-only, and read for one reason: the catalogue cache watches this file's
      # identity to know when the nightly publish has replaced the lake underneath
      # it. Writable would let this container take DuckDB's write lock and stall the
      # nightly job, which is the same reason datalake-mcp mounts it this way.
      - warehouse:/warehouse:ro
      - exports:/exports
      # The audit log, on its own volume. Separate from the warehouse on purpose:
      # the record of what a statement did must survive the statement, and the
      # warehouse volume is mounted read-only here anyway.
      - audit:/audit

  dagster-daemon:
    build: .
    container_name: datalake-daemon
    restart: unless-stopped
    env_file: [.env]
    command: ["dagster-daemon", "run", "-w", "/app/workspace.yaml"]
    # Two networks. `default` is this project's own; `primary_reseau` is where the
    # shared Postgres lives, and the LiteLLM ingest reads its spend ledger by
    # container name. Declared here rather than attached by hand with
    # `docker network connect`, which is how it was done before: a hand-made
    # attachment does not survive `compose up`, and the one that vanished took the
    # LiteLLM source down for two days without a sound. See IMPLEMENTATION_PLAN §8a.
    networks: [default, primary_reseau]
    volumes:
      - dagster_home:/dagster_home
      - warehouse:/warehouse
      # dagster.yaml is Dagster's own instance config file. It takes no flag and is
      # auto-discovered at $DAGSTER_HOME/dagster.yaml, so it has to land inside the
      # dagster_home volume's path rather than beside the code -- hence a bind mount
      # layered over the volume mount above.
      #
      # Deliberately per-service, and deliberately NOT written into the volume
      # itself: datalake-api and datalake-mcp mount the same dagster_home volume and
      # never open a DagsterInstance, so leaving the file out of the volume is what
      # keeps this change invisible to them.
      #
      # Read-only, because nothing running in this container should be able to
      # rewrite its own storage configuration.
      - ./dagster.yaml:/dagster_home/dagster.yaml:ro
      # workspace.yaml lists the TWO code locations `-w` loads (see that file):
      # this project's own module, plus ai_automations, served over gRPC by its
      # own container on primary_reseau. Read-only for the same reason
      # dagster.yaml is -- nothing in here should be able to widen the set of
      # code locations it loads.
      - ./workspace.yaml:/app/workspace.yaml:ro
      - ${PRIMARY_LOG_DIR:-/srv/npm/data/logs}:/logs/primary:ro
      - geoip:/geoip:ro
      # Sources added 2026-09-15, every one read-only (docs/METRICS_STREAMING_PLAN.md).
      # /var/log for the journal and auth.log. Radarr's and Sonarr's whole config
      # directories, because SQLite keeps -wal and -shm files beside the database and
      # they must be copied with it; config.xml in there holds an API key the pipeline
      # never opens. Jellyseerr's db/ directory only. The status page's data volume.
      - ${HOST_LOG_MOUNT:-/var/log}:/host/log:ro
      - ${RADARR_CONFIG_DIR:-/srv/media/Radarr/config}:/sources/radarr:ro
      - ${SONARR_CONFIG_DIR:-/srv/media/Sonarr/config}:/sources/sonarr:ro
      - ${JELLYSEERR_DB_DIR:-/srv/media/Jellyseer/config/db}:/sources/jellyseerr:ro
      - status-page-data:/sources/status-page:ro
      # See the matching mount on dagster-webserver above for why this is here
      # too: the daemon is what actually runs npm_daily/mcp_api_daily on their
      # schedules, not just the webserver's manual materialize button.
      - ${MCP_API_LOG_DIR_HOST:-/srv/ai-platform/deployment/logs/mcp-api}:/logs/mcp-api:ro
    environment:
      # Prometheus's published port, reached from this container over the LAN address.
      PROMETHEUS_URL: ${PROMETHEUS_URL:-http://198.51.100.26:9090}

volumes:
  dagster_home:
  warehouse:
  geoip:
  exports:
  audit:
  # The status page's own data volume, created by its compose project
  # (Sites perso/status-page). Mounted read-only; this project never owns it.
  status-page-data:
    external: true
    name: status-page_status-page-data

# An explicit subnet, because Docker's default address pools are fully allocated on
# this host: all fifteen private network ranges blocks and all fifteen private networks/20 blocks
# are taken by the 34 networks already running. Without this, `compose up` fails with
# "could not find an available, non-overlapping IPv4 address pool".
#
# 192.0.2.0/24 is free -- 198.51.100.0/24 and 203.0.113.0/29 are in use, and the LAN is
# 198.51.100.0/24. A /24 is ample for two containers and leaves the rest of 10/8 alone.
# The durable fix is a default-address-pools entry in /etc/docker/daemon.json, which
# needs root and a daemon restart; this does not.
networks:
  default:
    ipam:
      config:
        - subnet: 192.0.2.0/24

  # The shared Postgres network, created by the ai-platform stack. `external: true`
  # means compose joins it and never creates or removes it -- this project is a guest
  # there. If the ai-platform stack is down, `compose up` fails loudly with "network
  # primary_reseau declared as external, but could not be found", which is the correct
  # outcome: a warehouse that starts without its source would write empty days.
  primary_reseau:
    external: true