services: # MinIO - our local data lake storage (S3-compatible) minio: image: minio/minio:latest ports: - "9000:9000" # API port - "9001:9001" # Web console environment: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin123 command: server /data --console-address ":9001" volumes: - minio_data:/data healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] interval: 30s timeout: 10s retries: 3 # MinIO init — creates the required buckets on first startup minio-init: image: minio/minio:latest depends_on: minio: condition: service_healthy entrypoint: /bin/sh command: - -c - | mc alias set local http://minio:9000 minioadmin minioadmin123 && mc mb --ignore-existing local/healthcare && mc mb --ignore-existing local/healthcare-checkpoints && echo "Buckets ready." # Kafka + Zookeeper - for streaming vital signs zookeeper: image: confluentinc/cp-zookeeper:7.4.0 environment: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_TICK_TIME: 2000 kafka: image: confluentinc/cp-kafka:7.4.0 depends_on: [zookeeper] ports: - "9092:9092" environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" healthcheck: test: ["CMD", "kafka-topics", "--bootstrap-server", "localhost:9092", "--list"] interval: 30s timeout: 10s retries: 5 # PostgreSQL - metadata store for Airflow + data warehouse for dbt Gold layer postgres: image: postgres:15 ports: - "5432:5432" environment: POSTGRES_USER: airflow POSTGRES_PASSWORD: airflow POSTGRES_DB: airflow volumes: - postgres_data:/var/lib/postgresql/data - ./scripts/init_db.sql:/docker-entrypoint-initdb.d/init_db.sql healthcheck: test: ["CMD-SHELL", "pg_isready -U airflow"] interval: 10s timeout: 5s retries: 5 # Airflow - pipeline orchestration airflow-init: image: apache/airflow:2.8.0 depends_on: postgres: condition: service_healthy environment: AIRFLOW__CORE__EXECUTOR: LocalExecutor AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow AIRFLOW__CORE__LOAD_EXAMPLES: 'false' AIRFLOW__CORE__FERNET_KEY: 'ZmDfcTF7_60GrrY167zsiPd67pEvs0aGOv2oasOM1Pg=' entrypoint: /bin/bash command: - -c - | airflow db migrate && airflow users create \ --username admin \ --password admin \ --firstname Admin \ --lastname User \ --role Admin \ --email admin@example.com airflow-webserver: image: apache/airflow:2.8.0 depends_on: postgres: condition: service_healthy airflow-init: condition: service_completed_successfully ports: - "8080:8080" environment: AIRFLOW__CORE__EXECUTOR: LocalExecutor AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow AIRFLOW__CORE__LOAD_EXAMPLES: 'false' AIRFLOW__CORE__FERNET_KEY: 'ZmDfcTF7_60GrrY167zsiPd67pEvs0aGOv2oasOM1Pg=' volumes: - ./dags:/opt/airflow/dags - ./spark_jobs:/opt/airflow/spark_jobs - ./data:/opt/airflow/data command: webserver healthcheck: test: ["CMD", "curl", "--fail", "http://localhost:8080/health"] interval: 30s timeout: 10s retries: 5 airflow-scheduler: image: apache/airflow:2.8.0 depends_on: postgres: condition: service_healthy airflow-init: condition: service_completed_successfully environment: AIRFLOW__CORE__EXECUTOR: LocalExecutor AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow AIRFLOW__CORE__LOAD_EXAMPLES: 'false' AIRFLOW__CORE__FERNET_KEY: 'ZmDfcTF7_60GrrY167zsiPd67pEvs0aGOv2oasOM1Pg=' volumes: - ./dags:/opt/airflow/dags - ./spark_jobs:/opt/airflow/spark_jobs - ./data:/opt/airflow/data command: scheduler # R Shiny Dashboard - serves the analytics dashboard on port 3838 # Access at: http://localhost:3838/healthcare shiny-dashboard: build: context: ./shiny_app dockerfile: Dockerfile.shiny platform: linux/amd64 # Rosetta 2 emulation on Apple Silicon (M1/M2/M3) ports: - "3838:3838" volumes: # Mount Gold-layer CSVs + audit log read-only into the container - ./data/output:/data/output:ro environment: HEALTHCARE_DATA_PATH: /data/output restart: unless-stopped # No dependencies — runs standalone, reads local CSV files directly volumes: minio_data: postgres_data: