# Local end-to-end demo stack for feast-kafka-connect-sink.
#
# Prerequisite — supply the connector plugin (built in the connector repo) into ./plugin/:
#   unzip /path/to/feast-kafka-connect-sink-<version>.zip -d plugin/
#   ./scripts/build-kafka-source-id-smt.sh
#
# Then, from this directory:
#   docker compose up --build -d kafka feast connect
#   ./scripts/register-connector.sh json
#   docker compose --profile replay up --build producer
#   ./scripts/get-online-features_with_kafka_source_id.sh yellow_2024_01_000000
#
# See README.md for the full walkthrough.

name: feast-kafka-connect-sink-examples

services:
  kafka:
    image: apache/kafka:4.3.0
    hostname: kafka
    container_name: feast-demo-kafka
    ports:
      - "9092:9092"
      - "29092:29092"
    environment:
      KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m"
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093,PLAINTEXT_HOST://0.0.0.0:29092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      CLUSTER_ID: 5L6g3nShT-eMCtK--X86sw
    healthcheck:
      test: ["CMD-SHELL", "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1 || exit 1"]
      interval: 10s
      timeout: 10s
      retries: 15

  feast:
    build:
      context: ./feast
    hostname: feast
    container_name: feast-demo-feast
    ports:
      - "6566:6566"
    healthcheck:
      test: ["CMD-SHELL", "python -c \"import urllib.request,sys; urllib.request.urlopen('http://localhost:6566/health'); \" >/dev/null 2>&1 || exit 1"]
      interval: 10s
      timeout: 10s
      retries: 15

  connect:
    build:
      context: .
      dockerfile: connect/Dockerfile
    hostname: connect
    container_name: feast-demo-connect
    environment:
      KAFKA_HEAP_OPTS: "-Xms128m -Xmx512m"
    depends_on:
      kafka:
        condition: service_healthy
    ports:
      - "8083:8083"
    volumes:
      - ./connect/connect-distributed.properties:/config/connect-distributed.properties:ro
    entrypoint: ["/opt/kafka/bin/connect-distributed.sh", "/config/connect-distributed.properties"]
    healthcheck:
      test: ["CMD-SHELL", "wget -q -O - http://localhost:8083/connectors >/dev/null 2>&1 || exit 1"]
      interval: 10s
      timeout: 10s
      retries: 20

  # Replay producer: only runs with `--profile replay` so you control when the stream starts.
  producer:
    build:
      context: ./producer
    container_name: feast-demo-producer
    profiles: ["replay"]
    depends_on:
      kafka:
        condition: service_healthy
    environment:
      BOOTSTRAP_SERVERS: kafka:9092
      TOPIC: taxi-trips
      ROW_LIMIT: "5000"
      EMIT_RATE: "200"

  # --- Avro path (only runs with `--profile avro`) -------------------------------------------
  # Confluent Schema Registry. The connector itself decodes the Confluent-framed Avro bytes
  # (value.converter=ByteArrayConverter), so no converter JARs are added to the connect worker.
  # cp-schema-registry is a plain Kafka client + REST server, so it runs against the apache/kafka
  # broker. Match the Confluent Platform tag to your Kafka version (CP 8.x aligns with Kafka 4.x).
  schema-registry:
    image: confluentinc/cp-schema-registry:8.0.0
    hostname: schema-registry
    container_name: feast-demo-schema-registry
    profiles: ["avro", "protobuf"]
    depends_on:
      kafka:
        condition: service_healthy
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9092
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
    healthcheck:
      test: ["CMD-SHELL", "curl -sf http://localhost:8081/subjects >/dev/null 2>&1 || exit 1"]
      interval: 10s
      timeout: 10s
      retries: 20

  # Produces NYC-taxi events as Confluent-framed Avro to taxi-trips-avro.
  avro-producer:
    build:
      context: ./avro-producer
    container_name: feast-demo-avro-producer
    profiles: ["avro"]
    depends_on:
      kafka:
        condition: service_healthy
      schema-registry:
        condition: service_healthy
    environment:
      BOOTSTRAP_SERVERS: kafka:9092
      SCHEMA_REGISTRY_URL: http://schema-registry:8081
      TOPIC: taxi-trips-avro
      ROW_LIMIT: "5000"
      EMIT_RATE: "200"

  # Produces NYC-taxi events as Confluent-framed Protobuf to taxi-trips-protobuf.
  protobuf-producer:
    build:
      context: ./protobuf-producer
    container_name: feast-demo-protobuf-producer
    profiles: ["protobuf"]
    depends_on:
      kafka:
        condition: service_healthy
      schema-registry:
        condition: service_healthy
    environment:
      BOOTSTRAP_SERVERS: kafka:9092
      SCHEMA_REGISTRY_URL: http://schema-registry:8081
      TOPIC: taxi-trips-protobuf
      ROW_LIMIT: "5000"
      EMIT_RATE: "200"