version: "2.4"
services:

  zookeeper-1:
    image: confluentinc/cp-zookeeper
    hostname: zookeeper-1
    ports:
      - 12181:2181
    environment:
      ZOOKEEPER_SERVER_ID: "1"
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888
      ZOOKEEPER_4LW_COMMANDS_WHITELIST: "*"
      EXTRA_ARGS: -javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.12.0.jar=1234:/usr/share/jmx_exporter/zookeeper.yml
      KAFKA_HEAP_OPTS: "-Xmx512M -Xms512M"
    volumes:
    - ./jmx-exporter:/usr/share/jmx_exporter/

  zookeeper-2:
    image: confluentinc/cp-zookeeper
    hostname: zookeeper-2
    ports:
      - 12182:2181
    environment:
      ZOOKEEPER_SERVER_ID: "2"
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888
      ZOOKEEPER_4LW_COMMANDS_WHITELIST: "*"
      EXTRA_ARGS: -javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.12.0.jar=1234:/usr/share/jmx_exporter/zookeeper.yml
      KAFKA_HEAP_OPTS: "-Xmx512M -Xms512M"
    volumes:
    - ./jmx-exporter:/usr/share/jmx_exporter/

  zookeeper-3:
    image: confluentinc/cp-zookeeper
    hostname: zookeeper-3
    ports:
      - 12183:2181
    environment:
      ZOOKEEPER_SERVER_ID: "3"
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888
      ZOOKEEPER_4LW_COMMANDS_WHITELIST: "*"
      EXTRA_ARGS: -javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.12.0.jar=1234:/usr/share/jmx_exporter/zookeeper.yml
      KAFKA_HEAP_OPTS: "-Xmx512M -Xms512M"
    volumes:
    - ./jmx-exporter:/usr/share/jmx_exporter/

  kafka-1:
    image: confluentinc/cp-kafka:5.5.1
    hostname: kafka-1
    ports:
      - 19092:19092
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    environment:
      KAFKA_LISTENERS: "INTERNAL://kafka-1:9092,EXTERNAL://:19092"
      KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka-1:9092,EXTERNAL://localhost:19092"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT"
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
      KAFKA_ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181"
      KAFKA_BROKER_ID: "1"
      KAFKA_OPTS: -javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.12.0.jar=1234:/usr/share/jmx_exporter/kafka-broker.yml
      KAFKA_HEAP_OPTS: "-Xmx512M -Xms512M"
      KAFKA_MIN_INSYNC_REPLICAS: 2
    volumes:
    - ./jmx-exporter:/usr/share/jmx_exporter/

  kafka-2:
    image: confluentinc/cp-kafka:5.5.1
    hostname: kafka-2
    ports:
      - 19093:19093
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    environment:
      KAFKA_ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181"
      KAFKA_LISTENERS: "INTERNAL://kafka-2:9092,EXTERNAL://:19093"
      KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka-2:9092,EXTERNAL://localhost:19093"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT"
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
      KAFKA_BROKER_ID: "2"
      KAFKA_OPTS: -javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.12.0.jar=1234:/usr/share/jmx_exporter/kafka-broker.yml
      KAFKA_HEAP_OPTS: "-Xmx512M -Xms512M"
      KAFKA_MIN_INSYNC_REPLICAS: 2
    volumes:
    - ./jmx-exporter:/usr/share/jmx_exporter/

  kafka-3:
    image: confluentinc/cp-kafka:5.5.1
    hostname: kafka-3
    ports:
      - 19094:19094
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    environment:
      KAFKA_ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181"
      KAFKA_LISTENERS: "INTERNAL://kafka-3:9092,EXTERNAL://:19094"
      KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka-3:9092,EXTERNAL://localhost:19094"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT"
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
      KAFKA_BROKER_ID: "3"
      KAFKA_OPTS: -javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.12.0.jar=1234:/usr/share/jmx_exporter/kafka-broker.yml
      KAFKA_HEAP_OPTS: "-Xmx512M -Xms512M"
      KAFKA_MIN_INSYNC_REPLICAS: 2
    volumes:
    - ./jmx-exporter:/usr/share/jmx_exporter/

  producer:
    build: ./sample-application/producer
    environment:
      JAVA_OPTS: "-javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.12.0.jar=1234:/usr/share/jmx_exporter/kafka-producer.yml -Xmx256M -Xms256M"
      KAFKA_BOOTSTRAP_SERVERS: "kafka-1:9092, kafka-2:9092, kafka-3:9092"
      TOPIC: "sample"
      REPLICATION_FACTOR: 3
      NUMBER_OF_PARTITIONS: 2
      MESSAGE_BACKOFF: 1000 # Frequency of message injection
      KAFKA_ACKS: "all" # default: "1"
      #KAFKA_ENABLE_IDEMPOTENCE: "true"
      #KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION: "5" # default: 5
      #KAFKA_RETRIES: "10" # default: Integer.MAX_VALUE
      #KAFKA_REQUEST_TIMEOUT_MS: "900" # default: 30000
      #KAFKA_DELIVERY_TIMEOUT_MS: "1000" # default: 120000
      #KAFKA_TRANSACTIONAL_ID: "prod-1"
    volumes:
    - ./jmx-exporter:/usr/share/jmx_exporter/
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3

  consumer:
    build: ./sample-application/consumer
    hostname: consumer-1
    environment:
      JAVA_OPTS: -javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.12.0.jar=1234:/usr/share/jmx_exporter/kafka-consumer.yml -Xmx256M -Xms256M
      KAFKA_BOOTSTRAP_SERVERS: "kafka-1:9092, kafka-2:9092, kafka-3:9092"
      TOPIC: "sample"
      CHECK_GAPS: "false"
      COMMIT_STRATEGY: "AUTO_COMMIT" # possible values AUTO_COMMIT or PER_BATCH or PER_MESSAGES
      #KAFKA_ENABLE_AUTO_COMMIT_CONFIG: "false" # default true if group.id is provided
      #KAFKA_ISOLATION_LEVEL_CONFIG: "read_committed" # default read_uncommitted
      # default: org.apache.kafka.clients.consumer.RangeAssignor
      #KAFKA_PARTITION_ASSIGNMENT_STRATEGY_CONFIG: "org.apache.kafka.clients.consumer.CooperativeStickyAssignor"
    volumes:
    - ./jmx-exporter:/usr/share/jmx_exporter/
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3

  streams:
    build: ./sample-application/streams
    hostname: streams-1
    environment:
      JAVA_OPTS: -javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.12.0.jar=1234:/usr/share/jmx_exporter/kafka-streams-pre-2.5.yml -Xmx256M -Xms256M
      KAFKA_BOOTSTRAP_SERVERS: "kafka-1:9092, kafka-2:9092, kafka-3:9092"
      INPUT_TOPIC: "sample"
      OUTPUT_TOPIC: "count-even-odd-entries"
      REPLICATION_FACTOR: 3
      NUMBER_OF_PARTITIONS: 2
      KAFKA_PROCESSING_GUARANTEE: "exactly_once"
      KAFKA_METRICS_RECORDING_LEVEL: "DEBUG"
    volumes:
    - ./jmx-exporter:/usr/share/jmx_exporter/
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3

  ksqldb:
    image: confluentinc/ksqldb-server:0.13.0
    hostname: ksqldb
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
    ports:
      - 8088:8088
    environment:
      KSQL_LISTENERS: "http://0.0.0.0:8088"
      KSQL_BOOTSTRAP_SERVERS: "kafka-1:9092,kafka-2:9092,kafka-3:9092"
      KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
      KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"
      KSQL_OPTS: "-javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.12.0.jar=1234:/usr/share/jmx_exporter/ksqldb.yml"
      JMX_PORT: 1099
    volumes:
    - ./jmx-exporter:/usr/share/jmx_exporter/

  ksqldb-cli:
    image: confluentinc/ksqldb-cli:0.13.0
    container_name: ksqldb-cli
    depends_on:
      - ksqldb
    entrypoint: /bin/sh
    tty: true

  connect:
    image: confluentinc/cp-kafka-connect:5.5.0
    hostname: connect
    ports:
      - "8083:8083"
    environment:
      KAFKA_OPTS: -javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.12.0.jar=1234:/usr/share/jmx_exporter/kafka-connect.yml -Xmx256M -Xms256M
      CONNECT_BOOTSTRAP_SERVERS: "kafka-1:9092,kafka-2:9092,kafka-3:9092"
      CONNECT_REST_ADVERTISED_HOST_NAME: connect
      CONNECT_GROUP_ID: "connect-cluster"
      CONNECT_PRODUCER_CLIENT_ID: "connect-worker-producer"
      CONNECT_CONFIG_STORAGE_TOPIC: connect-configs
      CONNECT_OFFSET_STORAGE_TOPIC: connect-offsets
      CONNECT_STATUS_STORAGE_TOPIC: connect-status
      CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter"
      CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.storage.StringConverter"
      CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
      CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
    volumes:
    - ./jmx-exporter:/usr/share/jmx_exporter/
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3

  prometheus:
    image: prom/prometheus:v2.11.1
    ports:
      - 9090:9090
    volumes:
      - ./prometheus/:/etc/prometheus/

  alertmanager:
    image: prom/alertmanager:v0.18.0
    ports:
      - 9093:9093

  node-exporter:
    image: prom/node-exporter:v0.18.1
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - "--path.procfs=/host/proc"
      - "--path.sysfs=/host/sys"
      - "--collector.filesystem.ignored-mount-points"
      - "^(aufs|proc|nsfs|shm|cgroup|tmpfs|binfmt_misc|debugfs|devpts|fusectl|hugetlbfs|fuse.lxcfs|mqueue|pstore|securityfs|sysfs|autofs|devtmpfs|configfs)"

  kafka-lag-exporter:
    image: lightbend/kafka-lag-exporter:0.5.5
    hostname: kafka-lag-exporter
    restart: always
    ports:
      - 9999:9999
    volumes:
      - ./kafka-lag-exporter/:/opt/docker/conf/

  grafana:
    image: grafana/grafana:7.0.6
    environment:
      GF_INSTALL_PLUGINS: grafana-piechart-panel
    ports:
      - 3000:3000
    volumes:
      - ./grafana/provisioning/:/etc/grafana/provisioning/