name: obsingest

# Services: redpanda-0/1/2 + console, clickhouse, otel-collector,
# logs/traces/metrics-ingester, prometheus, grafana; superset+postgres+redis behind
# the "superset" profile.

x-redpanda-image: &redpanda-image docker.redpanda.com/redpandadata/redpanda:v24.2.7

services:
  # ---------- Redpanda 3-node cluster (Kafka API) ----------
  redpanda-0:
    image: *redpanda-image
    container_name: redpanda-0
    restart: unless-stopped
    command:
      - redpanda
      - start
      - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
      - --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
      - --rpc-addr redpanda-0:33145
      - --advertise-rpc-addr redpanda-0:33145
      - --mode dev-container
      - --smp 1
      - --memory 1G
      - --reserve-memory 0M
      - --default-log-level=info
      - --seeds redpanda-0:33145,redpanda-1:33145,redpanda-2:33145
    ports:
      - "19092:19092"
      - "9644:9644"
    volumes:
      - redpanda-0:/var/lib/redpanda/data
    healthcheck:
      test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 20
      start_period: 20s

  redpanda-1:
    image: *redpanda-image
    container_name: redpanda-1
    restart: unless-stopped
    command:
      - redpanda
      - start
      - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
      - --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092
      - --rpc-addr redpanda-1:33145
      - --advertise-rpc-addr redpanda-1:33145
      - --mode dev-container
      - --smp 1
      - --memory 1G
      - --reserve-memory 0M
      - --default-log-level=info
      - --seeds redpanda-0:33145,redpanda-1:33145,redpanda-2:33145
    ports:
      - "29092:29092"
    volumes:
      - redpanda-1:/var/lib/redpanda/data
    healthcheck:
      test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 20
      start_period: 20s
    depends_on: [redpanda-0]

  redpanda-2:
    image: *redpanda-image
    container_name: redpanda-2
    restart: unless-stopped
    command:
      - redpanda
      - start
      - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
      - --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092
      - --rpc-addr redpanda-2:33145
      - --advertise-rpc-addr redpanda-2:33145
      - --mode dev-container
      - --smp 1
      - --memory 1G
      - --reserve-memory 0M
      - --default-log-level=info
      - --seeds redpanda-0:33145,redpanda-1:33145,redpanda-2:33145
    ports:
      - "39092:39092"
    volumes:
      - redpanda-2:/var/lib/redpanda/data
    healthcheck:
      test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 20
      start_period: 20s
    depends_on: [redpanda-0]

  redpanda-console:
    image: docker.redpanda.com/redpandadata/console:v2.7.2
    container_name: redpanda-console
    environment:
      KAFKA_BROKERS: redpanda-0:9092,redpanda-1:9092,redpanda-2:9092
    ports:
      - "8090:8080"
    depends_on: [redpanda-0]

  # ---------- ClickHouse (store + query engine) ----------
  clickhouse:
    image: clickhouse/clickhouse-server:24.8
    container_name: clickhouse
    restart: unless-stopped
    environment:
      CLICKHOUSE_DB: obs
      CLICKHOUSE_USER: admin
      CLICKHOUSE_PASSWORD: pass
      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
    ports:
      - "8123:8123"
      - "9000:9000"
      - "9363:9363"
    volumes:
      - clickhouse-data:/var/lib/clickhouse
      # Mount the single file (NOT the dir) so the image's docker_related_config.xml
      # — which makes ClickHouse listen on 0.0.0.0 — is preserved.
      - ./deploy/clickhouse/config.d/prometheus.xml:/etc/clickhouse-server/config.d/prometheus.xml:ro
      - ./deploy/clickhouse/config.d/resources.xml:/etc/clickhouse-server/config.d/resources.xml:ro
    # Contain ClickHouse on the shared Colima VM (6 vCPU / 8 GiB) so its merge/insert
    # work can't starve Redpanda + the collector (the root cause of producer-side
    # drops). Largest single slice, but bounded: leaves 3 cores + ~5 GiB for the rest
    # of the stack. ClickHouse 24.8 reads this cgroup limit and self-sizes memory to it.
    deploy:
      resources:
        limits:
          cpus: "3.0"
          memory: 3g
    healthcheck:
      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8123/ping"]
      interval: 5s
      timeout: 3s
      retries: 20
      start_period: 10s
    ulimits:
      nofile:
        soft: 262144
        hard: 262144

  # ---------- OTel Collector (OTLP -> Redpanda) ----------
  # Scaled to 3 replicas — no container_name / host ports (a fixed host port can't be
  # published across replicas). Producers reach it over the compose network via the
  # `otel-collector` DNS name (round-robins across replicas); Prometheus scrapes :8888
  # internally via DNS SD. Host-side OTLP push to localhost:4317/4318 is no longer
  # available — load is generated inside the network (`make load`), so this is fine.
  otel-collector:
    image: otel/opentelemetry-collector-contrib:0.155.0
    restart: unless-stopped
    command: ["--config=/etc/otel/config.yaml"]
    deploy:
      replicas: 3
    volumes:
      - ./deploy/otel-collector/config.yaml:/etc/otel/config.yaml:ro
    depends_on:
      redpanda-0:
        condition: service_healthy

  # ---------- Go ingesters (Redpanda -> ClickHouse), one per signal ----------
  # ONE image (built once by `make ingester-image`) run as three roles, each scaled to
  # 3 replicas; each role picks its Cobra subcommand and joins its own consumer group
  # (obsingest-<signal>), so replicas split that topic's 12 partitions (4 each).
  # No container_name / host ports — a fixed host port can't be published across
  # replicas; Prometheus scrapes each replica's :8080 internally via DNS SD.
  # pull_policy:never -> use the locally-built image, never try to pull it.
  logs-ingester:
    image: obsingest-ingester:local
    pull_policy: never
    restart: unless-stopped
    command: ["logs"]
    deploy:
      replicas: 3
    environment: &ingester-env
      CH_HOST: clickhouse
      CH_PORT: "9000"
      CH_DB: obs
      CH_USER: admin
      CH_PASS: pass
      KAFKA_BROKERS: redpanda-0:9092,redpanda-1:9092,redpanda-2:9092
      CONSUMER_GROUP: obsingest
      # Low-latency batching: flush every 100ms (or 20k rows, whichever first).
      # Trades larger/less-frequent inserts for near-real-time processing latency.
      BATCH_MAX_WAIT: 100ms
      BATCH_MAX_ROWS: "20000"
    depends_on: &ingester-deps
      clickhouse:
        condition: service_healthy
      redpanda-0:
        condition: service_healthy

  traces-ingester:
    image: obsingest-ingester:local
    pull_policy: never
    restart: unless-stopped
    command: ["traces"]
    deploy:
      replicas: 3
    environment: *ingester-env
    depends_on: *ingester-deps

  metrics-ingester:
    image: obsingest-ingester:local
    pull_policy: never
    restart: unless-stopped
    command: ["metrics"]
    deploy:
      replicas: 3
    environment: *ingester-env
    depends_on: *ingester-deps

  # ---------- Monitoring plane (system observability) ----------
  prometheus:
    image: prom/prometheus:v2.55.1
    container_name: prometheus
    command:
      - --config.file=/etc/prometheus/prometheus.yml
    volumes:
      - ./deploy/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      # Read-only Docker socket for docker_sd_configs — lets Prometheus label the
      # scaled ingester replicas by container name instead of IP.
      - /var/run/docker.sock:/var/run/docker.sock:ro
    # Prometheus runs as `nobody`; add the docker.sock's group so it can read the socket.
    # 991 is the socket GID as seen inside a container on this (Colima) VM — override via
    # DOCKER_GID if yours differs: `docker run --rm -v /var/run/docker.sock:/s alpine stat -c %g /s`.
    group_add:
      - "${DOCKER_GID:-991}"
    ports:
      - "9090:9090"

  grafana:
    image: grafana/grafana:11.3.0
    container_name: grafana
    environment:
      GF_SECURITY_ADMIN_USER: admin
      GF_SECURITY_ADMIN_PASSWORD: admin
      GF_USERS_ALLOW_SIGN_UP: "false"
    volumes:
      - ./deploy/grafana/provisioning:/etc/grafana/provisioning:ro
      - ./deploy/grafana/dashboards:/var/lib/grafana/dashboards:ro
    ports:
      - "3000:3000"
    depends_on: [prometheus]

  # ---------- Superset (data-plane exploration) — profile: superset ----------
  superset-db:
    image: postgres:16-alpine
    container_name: superset-db
    profiles: ["superset"]
    environment:
      POSTGRES_USER: superset
      POSTGRES_PASSWORD: superset
      POSTGRES_DB: superset
    volumes:
      - superset-db:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U superset"]
      interval: 5s
      timeout: 3s
      retries: 20

  superset-redis:
    image: redis:7-alpine
    container_name: superset-redis
    profiles: ["superset"]

  superset-init:
    build: ./deploy/superset
    container_name: superset-init
    profiles: ["superset"]
    environment:
      SUPERSET_SECRET_KEY: obsingest-dev-secret-change-me
    volumes:
      - ./deploy/superset:/app/pythonpath:ro
    command:
      - /bin/sh
      - -c
      - |
        superset db upgrade &&
        superset fab create-admin --username admin --firstname admin --lastname admin --email admin@obsingest.local --password admin || true &&
        superset init &&
        python /app/pythonpath/add_clickhouse_db.py &&
        (python /app/pythonpath/create_dashboard.py || echo "dashboard provisioning skipped (is ClickHouse migrated?)")
    depends_on:
      superset-db:
        condition: service_healthy
      clickhouse:
        condition: service_healthy

  superset:
    build: ./deploy/superset
    container_name: superset
    profiles: ["superset"]
    environment:
      SUPERSET_SECRET_KEY: obsingest-dev-secret-change-me
    volumes:
      - ./deploy/superset:/app/pythonpath:ro
    ports:
      - "8088:8088"
    depends_on:
      superset-init:
        condition: service_completed_successfully
      superset-redis:
        condition: service_started

volumes:
  redpanda-0:
  redpanda-1:
  redpanda-2:
  clickhouse-data:
  superset-db: