services:
  # Kafka Brokers
  kafka0:
    # Configuration for Kafka Broker 0
    image: confluentinc/cp-kafka:7.6.1
    hostname: kafka0
    container_name: kafka0
    ports:
      - 9092:9092
      - 9997:9997
    environment:
      KAFKA_SOCKET_SEND_BUFFER_BYTES: 1024000
      KAFKA_SOCKET_RECEIVE_BUFFER_BYTES: 1024000
      KAFKA_LOG_FLUSH_INTERVAL_MESSAGES: 10000
      KAFKA_LOG_FLUSH_INTERVAL_MS: 1000
      CLUSTER_ID: fYJ1ooafQ6uBk6s_N3sYhA
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka0:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_PROCESS_ROLES: 'broker,controller'
      KAFKA_NODE_ID: 1
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka0:29093'
      KAFKA_LISTENERS: 'PLAINTEXT://kafka0:29092,CONTROLLER://kafka0:29093,PLAINTEXT_HOST://0.0.0.0:9092'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
      KAFKA_JMX_PORT: 9997
      KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka0 -Dcom.sun.management.jmxremote.rmi.port=9997
    networks:
      - acme

  # Schema Registry
  schema-registry0:
    # Configuration for Schema Registry
    image: confluentinc/cp-schema-registry:7.6.1
    ports:
      - 8085:8085
    hostname: schema-registry0
    container_name: schema-registry0
    depends_on:
      - kafka0
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka0:29092
      SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT
      SCHEMA_REGISTRY_HOST_NAME: schema-registry0
      SCHEMA_REGISTRY_LISTENERS: http://schema-registry0:8085

      SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "http"
      SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
      SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
    networks:
      - acme

  # Kafka Connect
  kafka-connect-builder:
    # Configuration for Kafka Connect Builder
    container_name: kafka-connect-builder
    build:
      context: .
      dockerfile: ./docker/kafka-connect/Dockerfile
    image: acme-kafka-connect

  kafka-connect0:
    # Configuration for Kafka Connect 0
    image: acme-kafka-connect
    ports:
      - 8083:8083
    hostname: kafka-connect0
    container_name: kafka-connect0
    depends_on:
      - kafka-connect-builder
      - kafka0
      - schema-registry0
    environment:
      CONNECT_BOOTSTRAP_SERVERS: "kafka0:29092" 
      CONNECT_GROUP_ID: compose-connect-group
      CONNECT_CONFIG_STORAGE_TOPIC: _connect_configs
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_OFFSET_STORAGE_TOPIC: _connect_offset
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_STATUS_STORAGE_TOPIC: _connect_status
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry0:8085
      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.storage.StringConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry0:8085
      CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_REST_ADVERTISED_HOST_NAME: kafka-connect0
      CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components,/usr/share/filestream-connectors,/tmp/kfk"
    volumes:
      - /tmp/kfk:/tmp/kfk:ro
      - /tmp/kfk/test.txt:/tmp/kfk/test.txt
    networks:
      - acme

  # REST Proxy
  rest-proxy:
    # Configuration for Kafka REST Proxy
    image: confluentinc/cp-kafka-rest:7.6.1
    depends_on:
      - kafka0
    ports:
      - 8082:8082
    hostname: rest-proxy
    container_name: rest-proxy
    environment:
      KAFKA_REST_HOST_NAME: rest-proxy
      KAFKA_REST_BOOTSTRAP_SERVERS: 'kafka0:29092'
      KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
    networks:
      - acme

  # ksqlDB Server
  ksqldb-server:
    # Configuration for ksqlDB Server
    image: confluentinc/ksqldb-server:0.29.0
    hostname: ksqldb-server
    container_name: ksqldb-server
    depends_on:
      - kafka0
      - schema-registry0
    ports:
      - "8088:8088"
    environment:
      KSQL_LISTENERS: http://0.0.0.0:8088
      KSQL_BOOTSTRAP_SERVERS: kafka0:29092
      KSQL_KSQL_SCHEMA_REGISTRY_URL: http://schema-registry0:8085
      KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
      KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"
    networks:
      - acme

  # Kafka UI
  kafka-ui:
    # Configuration for Kafka UI
    container_name: kafka-ui
    image: provectuslabs/kafka-ui:latest
    ports:
      - 8080:8080
    depends_on:
      - kafka0
      - schema-registry0
      - kafka-connect0
    environment:
      KAFKA_CLUSTERS_0_NAME: local
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:29092
      KAFKA_CLUSTERS_0_METRICS_PORT: 9997
      KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schema-registry0:8085
      KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: first
      KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://kafka-connect0:8083
      KAFKA_CLUSTERS_0_KSQLDBSERVER: http://ksqldb-server:8088
    networks:
      - acme

  # Minio
  minio:
    # Configuration for Minio
    image: minio/minio
    ports:
      - '9000:9000'
      - '9001:9001'
    hostname: minio-storage
    container_name: minio-storage
    volumes:
      - 'minio_data:/data'
    environment:
      MINIO_ROOT_USER: minioadmin
      MINIO_ROOT_PASSWORD: minioadmin
      MINIO_DEFAULT_BUCKETS: acme.eu-west-1.stg.data.lake
    command:
      # sh -c 'mc alias set minio http://minio:9000 minioadmin minioadmin &&
      server --console-address ":9001" /data
    networks:
      - acme
      # mc mb minio/acme.eu-west-1.stg.data.lake &&

  # Spark
  spark-builder:
    # Configuration for Spark Builder
    container_name: spark-builder
    build:
      context: .
      dockerfile: ./docker/spark/Dockerfile
    image: acme-spark

  # Spark Master
  spark-master:
    image: acme-spark
    container_name: spark-master
    ports:
      - "18080:8080"
      - "7077:7077"
      - "4040-4080:4040-4080"
    environment:
      SPARK_UI_PORT: 8080
      SPARK_MODE: master
      MINIO_ROOT_USER: minioadmin
      MINIO_ROOT_PASSWORD: minioadmin
      MINIO_URL: "minio-storage:9000"
    command: bin/spark-class org.apache.spark.deploy.master.Master
    volumes:
      - ./pyspark/apps:/opt/spark-apps
      - ./pyspark/data:/opt/spark-data
      - spark-checkpoints:/opt/spark-checkpoints
    networks:
      - acme

  # Spark Workers
  spark-worker-1:
    # Configuration for Spark Worker 1
    image: acme-spark
    container_name: spark-worker-1
    depends_on:
      - spark-builder
      - spark-master
    ports:
      - "28081:8081"
      - "7001:7001"
    environment:
      SPARK_MODE: worker
      SPARK_MASTER_URL: spark://spark-master:7077
      SPARK_WORKER_CORES: 2
      SPARK_WORKER_MEMORY: 2g
      SPARK_DRIVER_MEMORY: 2g
      SPARK_EXECUTOR_MEMORY: 2g
      SPARK_WORKLOAD: worker
      SPARK_LOCAL_IP: spark-worker-1
    command:
      bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
    volumes:
      - ./pyspark/apps:/opt/spark-apps
      - ./pyspark/data:/opt/spark-data
    networks:
      - acme

  # spark-worker-2:
  #   # Configuration for Spark Worker 2
  #   image: acme-spark
  #   container_name: spark-worker-2
  #   ports:
  #     - "18082:8080"
  #     - "7002:7002"
  #   depends_on:
  #     - spark-builder
  #     - spark-master
  #   environment:
  #     SPARK_MODE: worker
  #     SPARK_MASTER_URL: spark://spark-master:7077
  #     SPARK_WORKER_CORES: 2
  #     SPARK_WORKER_MEMORY: 2g
  #     SPARK_DRIVER_MEMORY: 2g
  #     SPARK_EXECUTOR_MEMORY: 2g
  #     SPARK_WORKLOAD: worker
  #     SPARK_LOCAL_IP: spark-worker-2
  #   command:
  #     bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
  #   volumes:
  #     - ./pyspark/apps:/opt/spark-apps
  #     - ./pyspark/data:/opt/spark-data
  #   networks:
  #     - acme

volumes:
  minio_data:
    # Configuration for MinIO data volume
    driver: local
  spark-checkpoints:
    driver: local

networks:
  acme:
    # Configuration for custom network
    driver: bridge