# duckWatch — single DuckLake + its observability plane (build spec §3).
#
# The write path (loadgen -> DuckDB/DuckLake -> Parquet + Postgres) lives on its
# own `isolated` network; only Postgres ALSO joins the `shared` network — the one
# deliberate bridge — so Debezium and the collector can reach its catalog. The
# rest (Kafka, Connect, ClickHouse, collector) observe from the shared plane.
#
# Superset lives behind the `superset` profile so it doesn't run during load tests.

services:
  postgres:
    image: postgres:16
    container_name: duckwatch-postgres
    # wal_level=logical + replication slots for Debezium CDC (spec §2.3).
    command:
      - postgres
      - -c
      - wal_level=logical
      - -c
      - max_wal_senders=10
      - -c
      - max_replication_slots=10
    environment:
      POSTGRES_USER: duckwatch
      POSTGRES_PASSWORD: duckwatch
      POSTGRES_DB: catalog
    networks:
      isolated:
        aliases: ["postgres"]
      shared:
        aliases: ["postgres"]
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U duckwatch -d catalog"]
      interval: 10s
      timeout: 5s
      retries: 10

  loadgen:
    build:
      context: .
      dockerfile: scripts/loadgen.Dockerfile
    image: duckwatch-loadgen:local
    container_name: duckwatch-loadgen
    depends_on:
      postgres:
        condition: service_healthy
    environment:
      RPS: ${RPS:-1}
      DURATION: ${DURATION:-300}
      WRITERS: ${WRITERS:-3}
      OFFSET_STRIDE: ${OFFSET_STRIDE:-100000}
      PGUSER: duckwatch
      PGPASSWORD: duckwatch
    volumes:
      - tpch-source-vol:/data/tpch:ro
      - lake-data:/data
    networks: [isolated]                # write path stays isolated
    restart: on-failure


  kafka:
    image: apache/kafka:3.9.0
    container_name: duckwatch-kafka
    networks: [shared]
    ports: ["9092:9092"]
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      # Short retention — ClickHouse is the system of record (spec §2.7).
      KAFKA_LOG_RETENTION_HOURS: 6
    healthcheck:
      test: ["CMD-SHELL", "/opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1"]
      interval: 10s
      timeout: 10s
      retries: 12

  connect:
    image: debezium/connect:2.7.3.Final
    container_name: duckwatch-connect
    networks: [shared]
    ports: ["8083:8083"]
    depends_on:
      kafka:
        condition: service_healthy
    environment:
      BOOTSTRAP_SERVERS: kafka:9092
      GROUP_ID: duckwatch-connect
      CONFIG_STORAGE_TOPIC: connect_configs
      OFFSET_STORAGE_TOPIC: connect_offsets
      STATUS_STORAGE_TOPIC: connect_statuses
      CONFIG_STORAGE_REPLICATION_FACTOR: 1
      OFFSET_STORAGE_REPLICATION_FACTOR: 1
      STATUS_STORAGE_REPLICATION_FACTOR: 1
      # Flat JSON (no schema envelope) so ClickHouse's Kafka engine reads JSONEachRow.
      KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      KEY_CONVERTER_SCHEMAS_ENABLE: "false"
      VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
    healthcheck:
      test: ["CMD-SHELL", "curl -sf http://localhost:8083/ || exit 1"]
      interval: 10s
      timeout: 10s
      retries: 18

  clickhouse:
    image: clickhouse/clickhouse-server:24.8
    container_name: duckwatch-clickhouse
    networks: [shared]
    ports: ["8123:8123", "9000:9000"]
    ulimits:
      nofile: { soft: 262144, hard: 262144 }
    environment:
      CLICKHOUSE_DB: duckwatch
      CLICKHOUSE_USER: admin
      CLICKHOUSE_PASSWORD: pass
      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
    volumes:
      - ./clickhouse/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
      - clickhouse-data:/var/lib/clickhouse
    healthcheck:
      test: ["CMD-SHELL", "clickhouse-client --user admin --password pass -q 'SELECT 1' || exit 1"]
      interval: 10s
      timeout: 10s
      retries: 12

  collector:
    build: .
    image: duckwatch:local
    container_name: duckwatch-collector
    command: ["collector"]
    networks: [shared]
    depends_on:
      kafka:
        condition: service_healthy
    environment:
      KAFKA_BROKERS: kafka:9092
      POSTGRES_HOST: postgres
      POSTGRES_DB: catalog
      POLL_INTERVAL: ${POLL_INTERVAL:-5s}
      POSTGRES_USER: duckwatch
      POSTGRES_PASSWORD: duckwatch
      POSTGRES_PORT: "5432"
    restart: on-failure

  superset-db:
    image: postgres:16-alpine
    profiles: ["superset"]
    networks: [shared]
    environment:
      POSTGRES_USER: superset
      POSTGRES_PASSWORD: superset
      POSTGRES_DB: superset
    volumes: ["superset-db:/var/lib/postgresql/data"]
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U superset"]
      interval: 10s
      timeout: 5s
      retries: 10

  superset-redis:
    image: redis:7-alpine
    profiles: ["superset"]
    networks: [shared]

  superset-init:
    build: ./deploy/superset
    image: duckwatch-superset:local
    profiles: ["superset"]
    networks: [shared]
    depends_on:
      superset-db:
        condition: service_healthy
      clickhouse:
        condition: service_healthy
    environment:
      SUPERSET_SECRET_KEY: duckwatch-dev-secret-change-me
    volumes:
      - ./deploy/superset:/app/pythonpath:ro
    command:
      - /bin/sh
      - -c
      - |
        superset db upgrade &&
        superset fab create-admin --username admin --firstname admin --lastname admin \
          --email admin@duckwatch.local --password admin || true &&
        superset init &&
        python /app/pythonpath/add_clickhouse_db.py &&
        (python /app/pythonpath/create_dashboard.py || echo "dashboard provisioning skipped (is ClickHouse migrated?)")

  superset:
    # Reuses the image built by superset-init (which always runs first via
    # depends_on). Declaring `build:` here too races the parallel bake exporter
    # for the same tag -> "image already exists".
    image: duckwatch-superset:local
    profiles: ["superset"]
    networks: [shared]
    ports: ["8088:8088"]
    depends_on:
      superset-init:
        condition: service_completed_successfully
      superset-redis:
        condition: service_started
    environment:
      SUPERSET_SECRET_KEY: duckwatch-dev-secret-change-me
    volumes:
      - ./deploy/superset:/app/pythonpath:ro
    command: ["/usr/bin/run-server.sh"]

networks:
  shared:
    name: duckwatch-shared
  # The write path (loadgen + Postgres) is isolated; only Postgres also joins shared.
  isolated:
    name: duckwatch-isolated

volumes:
  clickhouse-data:
  superset-db:
  lake-data:
  tpch-source-vol:
    name: tpch-source-vol