version: '3' # Configuration environment variables: # - DATAFLOW_VERSION and SKIPPER_VERSION specify what DataFlow and Skipper image versions to use. # - STREAM_APPS_URI and TASK_APPS_URI are used to specify what Stream and Task applications to pre-register. # - HOST_MOUNT_PATH and DOCKER_MOUNT_PATH are used to set the host and docker mount folders. # If not set HOST_MOUNT_PATH defaults to the local host folder where the docker compose is being started. # If not set DOCKER_MOUNT_PATH defaults to /home/cnb/scdf on dataflow-server and skipper containers. # Example to mount the local Maven repository: HOST_MOUNT_PATH=~/.m2/repository/ DOCKER_MOUNT_PATH=/root/.m2 docker-compose up # - APPS_PORT_RANGE allows you to override the port range exported by the Skipper server. By default the APPS_PORT_RANGE # must supersede the local deployer's PORTRANGE_LOW and PORTRANGE_HIGH - the ports assigned to the stream apps run inside the Skipper container. # The stream apps tha run in their own docker containers must set [DOCKER_PORTRANGE_LOW, DOCKER_PORTRANGE_HIGH] range that # doesn't overlap with the APPS_PORT_RANGE. # # Exposed container ports: # - 9393:9393 - Data Flow server port (http://localhost:9393/dashboard) # - 7577:7577 - Skipper server port (http://localhost:7577/api) # - 20000-20105:20000-20105 - Port range for all deployed stream applications that run inside the Skipper container (e.g. maven:// registered apps). # That means you can reach the application's actuator endpoints from your host machine. # The deployed stream applications that run in their own docker containers (e.g. docker:// registered apps), # can be reached on the the ports they expose. services: mysql: image: mysql:5.7.25 container_name: dataflow-mysql environment: MYSQL_DATABASE: dataflow MYSQL_USER: root MYSQL_ROOT_PASSWORD: rootpw expose: - 3306 kafka-broker: image: confluentinc/cp-kafka:5.5.2 container_name: dataflow-kafka expose: - "9092" environment: - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-broker:9092 - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_ADVERTISED_HOST_NAME=kafka-broker - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - KAFKA_LOG4J_ROOT_LOGLEVEL=ERROR - KAFKA_LOG4J_LOGGERS=org.apache.zookeeper=ERROR,org.apache.kafka=ERROR,kafka=ERROR,kafka.cluster=ERROR,kafka.controller=ERROR,kafka.coordinator=ERROR,kafka.log=ERROR,kafka.server=ERROR,kafka.zookeeper=ERROR,state.change.logger=ERROR depends_on: - zookeeper zookeeper: image: confluentinc/cp-zookeeper:5.5.2 container_name: dataflow-kafka-zookeeper expose: - "2181" environment: - ZOOKEEPER_CLIENT_PORT=2181 dataflow-server: image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:-2.8.1} container_name: dataflow-server ports: - "9393:9393" environment: - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS=PLAINTEXT://kafka-broker:9092 - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_STREAMS_BINDER_BROKERS=PLAINTEXT://kafka-broker:9092 - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_BINDER_ZKNODES=zookeeper:2181 - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_STREAMS_BINDER_ZKNODES=zookeeper:2181 - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_KAFKA_STREAMS_PROPERTIES_METRICS_RECORDING_LEVEL=DEBUG - SPRING_CLOUD_SKIPPER_CLIENT_SERVER_URI=${SKIPPER_URI:-http://skipper-server:7577}/api - SPRING_DATASOURCE_URL=jdbc:mysql://mysql:3306/dataflow - SPRING_DATASOURCE_USERNAME=root - SPRING_DATASOURCE_PASSWORD=rootpw - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver # (Optionally) authenticate the default Docker Hub access for the App Metadata access. - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_USER=${METADATA_DEFAULT_DOCKERHUB_USER} - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_SECRET=${METADATA_DEFAULT_DOCKERHUB_PASSWORD} depends_on: - kafka-broker - skipper-server restart: always volumes: - ${HOST_MOUNT_PATH:-.}:${DOCKER_MOUNT_PATH:-/home/cnb/scdf} app-import-stream: image: springcloud/baseimage:1.0.0 container_name: dataflow-app-import-stream depends_on: - dataflow-server command: > /bin/sh -c " ./wait-for-it.sh -t 360 dataflow-server:9393; wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-maven-latest&force=true}'; wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/3.0.1' --no-check-certificate --post-data='uri=maven://org.springframework.cloud.stream.app:log-sink-kafka:3.0.1'; wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps/sink/ver-log/2.1.5.RELEASE' --no-check-certificate --post-data='uri=maven://org.springframework.cloud.stream.app:log-sink-kafka:2.1.5.RELEASE'; echo 'Maven Stream apps imported'" app-import-task: image: springcloud/baseimage:1.0.0 container_name: dataflow-app-import-task depends_on: - dataflow-server command: > /bin/sh -c " ./wait-for-it.sh -t 360 dataflow-server:9393; wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${TASK_APPS_URI:-https://dataflow.spring.io/task-maven-latest&force=true}'; echo 'Maven Task apps imported'" skipper-server: image: springcloud/spring-cloud-skipper-server:${SKIPPER_VERSION:-2.7.1} container_name: skipper ports: - "7577:7577" - ${APPS_PORT_RANGE:-20000-20195:20000-20195} environment: - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_LOW=20000 - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_HIGH=20190 - SPRING_DATASOURCE_URL=jdbc:mysql://mysql:3306/dataflow - SPRING_DATASOURCE_USERNAME=root - SPRING_DATASOURCE_PASSWORD=rootpw - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver - LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_SKIPPER_SERVER_DEPLOYER=ERROR restart: always volumes: - ${HOST_MOUNT_PATH:-.}:${DOCKER_MOUNT_PATH:-/home/cnb/scdf}