--- version: '2.1' services: controller: image: ticdc:latest build: context: . dockerfile: ./Dockerfile.development volumes: - /data - ./docker/logs:/logs - ./docker/config:/config command: - /usr/bin/socat - -v - tcp-l:1234,fork - exec:'/bin/cat' ports: - "1234:1234" depends_on: - "upstream-pd" - "schema-registry" - "kafka-connect-01" - "kafka" - "capturer0" - "capturer1" - "capturer2" restart: on-failure capturer0: image: ticdc:latest build: context: . dockerfile: ./Dockerfile.development volumes: - /data - ./docker/logs:/logs entrypoint: "/cdc server" command: - --addr=0.0.0.0:8300 - --pd=http://upstream-pd:2379 - --log-file=/logs/capturer0.log - --log-level=debug - --advertise-addr=capturer0:8300 depends_on: - "upstream-tidb" - "downstream-tidb" - "kafka" restart: on-failure capturer1: image: ticdc:latest build: context: . dockerfile: ./Dockerfile.development volumes: - /data - ./docker/logs:/logs entrypoint: "/cdc server" command: - --addr=0.0.0.0:8300 - --pd=http://upstream-pd:2379 - --log-file=/logs/capturer1.log - --log-level=debug - --advertise-addr=capturer1:8300 depends_on: - "upstream-tidb" - "downstream-tidb" - "kafka" restart: on-failure capturer2: image: ticdc:latest build: context: . dockerfile: ./Dockerfile.development volumes: - /data - ./docker/logs:/logs entrypoint: "/cdc server" command: - --addr=0.0.0.0:8300 - --pd=http://upstream-pd:2379 - --log-file=/logs/capturer2.log - --log-level=debug - --advertise-addr=capturer2:8300 depends_on: - "upstream-tidb" - "downstream-tidb" - "kafka" restart: on-failure upstream-pd: image: pingcap/pd:release-4.0-nightly ports: - "2379:2379" volumes: - ./docker/config/pd.toml:/pd.toml:ro - /data - ./docker/logs:/logs command: - --name=upstream-pd - --client-urls=http://0.0.0.0:2379 - --peer-urls=http://0.0.0.0:2380 - --advertise-client-urls=http://upstream-pd:2379 - --advertise-peer-urls=http://upstream-pd:2380 - --initial-cluster=upstream-pd=http://upstream-pd:2380 - --data-dir=/data/upstream-pd - --config=/pd.toml - --log-file=/logs/upstream-pd.log - -L=debug restart: on-failure upstream-tikv0: image: pingcap/tikv:release-4.0-nightly volumes: - ./docker/config/tikv.toml:/tikv.toml:ro - /data - ./docker/logs:/logs command: - --addr=0.0.0.0:20160 - --advertise-addr=upstream-tikv0:20160 - --data-dir=/data/upstream-tikv0 - --pd=upstream-pd:2379 - --config=/tikv.toml - --log-file=/logs/upstream-tikv0.log - --log-level=debug depends_on: - "upstream-pd" restart: on-failure upstream-tikv1: image: pingcap/tikv:release-4.0-nightly volumes: - ./docker/config/tikv.toml:/tikv.toml:ro - /data - ./docker/logs:/logs command: - --addr=0.0.0.0:20160 - --advertise-addr=upstream-tikv1:20160 - --data-dir=/data/upstream-tikv1 - --pd=upstream-pd:2379 - --config=/tikv.toml - --log-file=/logs/upstream-tikv1.log - --log-level=debug depends_on: - "upstream-pd" restart: on-failure upstream-tikv2: image: pingcap/tikv:release-4.0-nightly volumes: - ./docker/config/tikv.toml:/tikv.toml:ro - /data - ./docker/logs:/logs command: - --addr=0.0.0.0:20160 - --advertise-addr=upstream-tikv2:20160 - --data-dir=/data/upstream-tikv2 - --pd=upstream-pd:2379 - --config=/tikv.toml - --log-file=/logs/upstream-tikv2.log - --log-level=debug depends_on: - "upstream-pd" restart: on-failure upstream-tidb: image: pingcap/tidb:release-4.0-nightly ports: - "4000:4000" - "10080:10080" volumes: - ./docker/config/tidb.toml:/tidb.toml:ro - ./docker/logs:/logs command: - --store=tikv - --path=upstream-pd:2379 - --config=/tidb.toml - --log-file=/logs/upstream-tidb.log - --advertise-address=upstream-tidb - -L=debug depends_on: - "upstream-tikv0" - "upstream-tikv1" - "upstream-tikv2" restart: on-failure downstream-pd: image: pingcap/pd:release-4.0-nightly ports: - "3379:2379" volumes: - ./docker/config/pd.toml:/pd.toml:ro - /data - ./docker/logs:/logs command: - --name=downstream-pd - --client-urls=http://0.0.0.0:2379 - --peer-urls=http://0.0.0.0:2380 - --advertise-client-urls=http://downstream-pd:2379 - --advertise-peer-urls=http://downstream-pd:2380 - --initial-cluster=downstream-pd=http://downstream-pd:2380 - --data-dir=/data/downstream-pd - --config=/pd.toml - --log-file=/logs/downstream-pd.log - -L=debug restart: on-failure downstream-tikv0: image: pingcap/tikv:release-4.0-nightly volumes: - ./docker/config/tikv.toml:/tikv.toml:ro - /data - ./docker/logs:/logs command: - --addr=0.0.0.0:20160 - --advertise-addr=downstream-tikv0:20160 - --data-dir=/data/downstream-tikv0 - --pd=downstream-pd:2379 - --config=/tikv.toml - --log-file=/logs/downstream-tikv0.log - --log-level=debug depends_on: - "downstream-pd" restart: on-failure downstream-tikv1: image: pingcap/tikv:release-4.0-nightly volumes: - ./docker/config/tikv.toml:/tikv.toml:ro - /data - ./docker/logs:/logs command: - --addr=0.0.0.0:20160 - --advertise-addr=downstream-tikv1:20160 - --data-dir=/data/downstream-tikv1 - --pd=downstream-pd:2379 - --config=/tikv.toml - --log-file=/logs/downstream-tikv1.log - --log-level=debug depends_on: - "downstream-pd" restart: on-failure downstream-tikv2: image: pingcap/tikv:release-4.0-nightly volumes: - ./docker/config/tikv.toml:/tikv.toml:ro - /data - ./docker/logs:/logs command: - --addr=0.0.0.0:20160 - --advertise-addr=downstream-tikv2:20160 - --data-dir=/data/downstream-tikv2 - --pd=downstream-pd:2379 - --config=/tikv.toml - --log-file=/logs/downstream-tikv2.log - --log-level=debug depends_on: - "downstream-pd" restart: on-failure downstream-tidb: image: pingcap/tidb:release-4.0-nightly ports: - "5000:4000" - "20080:10080" volumes: - ./docker/config/tidb.toml:/tidb.toml:ro - ./docker/logs:/logs command: - --store=tikv - --path=downstream-pd:2379 - --config=/tidb.toml - --log-file=/logs/downstream-tidb.log - --advertise-address=downstream-tidb - -L=debug depends_on: - "downstream-tikv0" - "downstream-tikv1" - "downstream-tikv2" restart: on-failure # The rest of the file is adapted from https://github.com/confluentinc/demo-scene/blob/master/connect-jdbc/docker-compose.yml zookeeper: image: confluentinc/cp-zookeeper:5.5.1 container_name: zookeeper environment: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_TICK_TIME: 2000 kafka: image: confluentinc/cp-enterprise-kafka:5.5.1 container_name: kafka depends_on: - zookeeper ports: # Exposes 9092 for external connections to the broker # Use kafka:29092 for connections internal on the docker network # See https://rmoff.net/2018/08/02/kafka-listeners-explained/ for details - 9092:9092 environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://kafka:9092 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100 CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:29092 CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181 CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1 CONFLUENT_METRICS_ENABLE: 'true' CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous' schema-registry: image: confluentinc/cp-schema-registry:5.5.1 container_name: schema-registry ports: - 8081:8081 depends_on: - zookeeper - kafka environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181 kafka-connect-01: image: confluentinc/cp-kafka-connect:5.5.1 container_name: kafka-connect-01 depends_on: - zookeeper - kafka - schema-registry - downstream-tidb ports: - 8083:8083 environment: CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n" CONNECT_BOOTSTRAP_SERVERS: "kafka:29092" CONNECT_REST_PORT: 8083 CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect-01" CONNECT_GROUP_ID: compose-connect-group CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081' CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081' CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO" CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR" CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1" CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1" CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1" CONNECT_PLUGIN_PATH: '/usr/share/java' command: - /bin/bash - -c - | # JDBC Drivers # ------------ # MySQL cd /usr/share/java/kafka-connect-jdbc/ wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.21.tar.gz tar -xf mysql-connector-java-8.0.21.tar.gz mv mysql-connector-java-8.0.21/mysql-connector-java-8.0.21.jar ./ # Now launch Kafka Connect sleep infinity & /etc/confluent/docker/run kafka-connect-healthcheck: image: devshawn/kafka-connect-healthcheck:0.1.0 container_name: kafka-connect-healthcheck depends_on: - kafka-connect-01 ports: - 18083:18083 environment: HEALTHCHECK_CONNECT_URL: 'http://kafka-connect-01:8083'