services: kafka1: build: context: ./resources/kafka dockerfile: Dockerfile container_name: kafka1 hostname: kafka1 ports: - "9092:9092" - "7071:7071" volumes: - ./monitoring/jmx-exporter:/opt/jmx-exporter/config environment: KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT' KAFKA_LISTENERS: 'INTERNAL://kafka1:29092,CONTROLLER://kafka1:29093,EXTERNAL://0.0.0.0:9092' KAFKA_ADVERTISED_LISTENERS: 'INTERNAL://kafka1:29092,EXTERNAL://localhost:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERNAL' KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka1:29093' KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 CLUSTER_ID: 'p8fFEbKGQ22B6M_Da_vCBw' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' # Prometheus JMX exporter (javaagent connects to JMX in-process, no separate JMX port needed) EXTRA_ARGS: >- -javaagent:/opt/jmx-exporter/jmx_prometheus_javaagent.jar=7071:/opt/jmx-exporter/config/kafka-broker.yml healthcheck: test: ["CMD", "kafka-topics", "--bootstrap-server", "kafka1:29092", "--list"] interval: 30s timeout: 5s retries: 5 start_period: 1m kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 # Changed to avoid port clash with akhq depends_on: - kafka1 - schema-registry environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: 'kafka1:29092' KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schema-registry:8081 KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: connect KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://connect:8083 DYNAMIC_CONFIG_ENABLED: 'true' schema-registry: build: context: ./resources/schema-registry dockerfile: Dockerfile container_name: schema-registry hostname: schema-registry ports: - "8081:8081" - "7073:7073" volumes: - ./monitoring/jmx-exporter:/opt/jmx-exporter/config environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka1:29092' SCHEMA_REGISTRY_LISTENERS: 'http://0.0.0.0:8081' # JMX + Prometheus exporter SCHEMA_REGISTRY_JMX_OPTS: >- -javaagent:/opt/jmx-exporter/jmx_prometheus_javaagent.jar=7073:/opt/jmx-exporter/config/schema-registry.yml depends_on: - kafka1 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8081"] interval: 30s timeout: 5s retries: 5 start_period: 1m connect: build: context: ./resources/kafka-connect dockerfile: Dockerfile hostname: connect container_name: connect ports: - 8083:8083 - 5555:5555 - 7072:7072 volumes: - ./schemas:/opt/kafka/schemas - ./monitoring/jmx-exporter:/opt/jmx-exporter/config environment: CONNECT_BOOTSTRAP_SERVERS: 'kafka1:29092' CONNECT_REST_ADVERTISED_HOST_NAME: "connect-s3" CONNECT_REST_PORT: 8083 CONNECT_GROUP_ID: connect-cluster-group CONNECT_CONFIG_STORAGE_TOPIC: _kafka-connect-configs CONNECT_OFFSET_STORAGE_TOPIC: _kafka-connect-offsets CONNECT_STATUS_STORAGE_TOPIC: _kafka-connect-status CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "false" CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false" CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter" CONNECT_INTERNAL_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" CONNECT_INTERNAL_KEY_CONVERTER_SCHEMAS_ENABLE: "false" CONNECT_INTERNAL_VALUE_CONVERTER_SCHEMAS_ENABLE: "false" CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO" CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR" CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1" CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1" CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1" CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components/,/connectors/,/custom-plugins/' CONNECT_CONNECTOR_CLIENT_CONFIG_OVERRIDE_POLICY: "All" AWS_ACCESS_KEY_ID: "minioadmin" AWS_SECRET_ACCESS_KEY: "minioadmin" AWS_REGION: "us-east-1" # JMX Configuration KAFKA_JMX_PORT: 5555 KAFKA_JMX_HOSTNAME: connect KAFKA_JMX_OPTS: >- -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=5555 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.rmi.port=5555 -Djava.rmi.server.hostname=connect -Dcom.sun.management.jmxremote.local.only=false # Prometheus JMX exporter (javaagent connects to JMX in-process) EXTRA_ARGS: >- -javaagent:/opt/jmx-exporter/jmx_prometheus_javaagent.jar=7072:/opt/jmx-exporter/config/kafka-connect.yml healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8083/connectors"] interval: 30s timeout: 5s retries: 5 start_period: 1m minio: image: minio/minio:RELEASE.2025-02-07T23-21-09Z ports: - "9000:9000" - "9001:9001" environment: MINIO_PROMETHEUS_AUTH_TYPE: public command: server --console-address ":9001" /data healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] interval: 30s timeout: 5s retries: 5 start_period: 1m kafka-producer: build: context: ./Kafka-Producer dockerfile: Dockerfile container_name: kafka-producer hostname: kafka-producer ports: - "8000:8000" depends_on: kafka1: condition: service_healthy schema-registry: condition: service_healthy prometheus: image: prom/prometheus:latest container_name: prometheus ports: - "9090:9090" volumes: - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml depends_on: - kafka1 - connect - schema-registry grafana: image: grafana/grafana:latest container_name: grafana ports: - "3000:3000" environment: GF_SECURITY_ADMIN_PASSWORD: admin GF_AUTH_ANONYMOUS_ENABLED: "true" GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer volumes: - ./monitoring/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources - ./monitoring/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards depends_on: - prometheus networks: default: name: my-network external: true