name: rsync-ai

# ─────────────────────────────────────────────────────────────────────────────
# Kafka client security, for a broker this file did not start.
#
# Point KAFKA_BROKERS at a managed cluster (MSK, Confluent Cloud, Aiven) and the
# clients below still default to PLAINTEXT unless these names reach them. Compose
# injects only what a service DECLARES, so a credential sitting in .env reaches
# interpolation but not the container: the operator sets a password, the client
# connects anonymously, and an ACL-enforcing broker rejects it with an error
# naming the BROKER rather than the missing setting.
#
# Every key defaults to empty and empty means "unset" -- kafkaclient reads
# `protocolWasSet := c.SecurityProtocol != ""` (shared/go/kafkaclient/config.go),
# so the bundled-broker default renders exactly as it did before these keys
# existed. Kept key-for-key in lockstep with the same anchor in
# docker-compose.quickstart.yml; tests/test_compose_kafka_security_env.py fails
# if the two drift.
# ─────────────────────────────────────────────────────────────────────────────
x-kafka-security: &kafka-security
  KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-}
  KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-}
  KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-}
  KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-}
  # OIDC client-credentials, for KAFKA_SASL_MECHANISM=OAUTHBEARER.
  KAFKA_SASL_OAUTHBEARER_CLIENT_ID: ${KAFKA_SASL_OAUTHBEARER_CLIENT_ID:-}
  KAFKA_SASL_OAUTHBEARER_CLIENT_SECRET: ${KAFKA_SASL_OAUTHBEARER_CLIENT_SECRET:-}
  KAFKA_SASL_OAUTHBEARER_TOKEN_ENDPOINT: ${KAFKA_SASL_OAUTHBEARER_TOKEN_ENDPOINT:-}
  KAFKA_SASL_OAUTHBEARER_SCOPE: ${KAFKA_SASL_OAUTHBEARER_SCOPE:-}
  KAFKA_SASL_OAUTHBEARER_EXTENSIONS: ${KAFKA_SASL_OAUTHBEARER_EXTENSIONS:-}
  # Re-permits a plain-http token endpoint off loopback, which every runtime
  # otherwise refuses at start-up (the client-credentials grant POSTs
  # CLIENT_SECRET on every fetch). Carried here for ONE reason: kafka-connect
  # and the kafka-init job read their whole security profile from this anchor,
  # so without a passthrough a rig with a throwaway http IdP can configure the
  # Go and Python services but not the JVM ones -- the same image behaving
  # differently under compose than under the chart, which does make it
  # reachable. The default is empty, i.e. unset, i.e. the refusal stays armed;
  # a passthrough is not a setting. Never give it a value in this file or in
  # docker-compose.prod.yml --
  # llm-service/tests/test_insecure_token_endpoint_rule_is_one_rule.py fails if
  # any file but the Kafka security matrix harness sets it to true.
  KAFKA_SASL_OAUTHBEARER_ALLOW_INSECURE_TOKEN_ENDPOINT: ${KAFKA_SASL_OAUTHBEARER_ALLOW_INSECURE_TOKEN_ENDPOINT:-}
  # JVM-only; in this file only kafka-connect has a JVM. Carried anyway because
  # Debezium's schema-history client is configured from the connector
  # container's environment, so the name has to survive the trip.
  KAFKA_SASL_OAUTHBEARER_LOGIN_CALLBACK_HANDLER: ${KAFKA_SASL_OAUTHBEARER_LOGIN_CALLBACK_HANDLER:-}
  # AWS_MSK_IAM. The Go services implement it (kafkaclient/tokenauth/msk.go); the
  # Python tier and the JVM Connect image do not, so an IAM-only cluster runs the
  # Go data plane and leaves the CDC profile unable to authenticate.
  KAFKA_AWS_REGION: ${KAFKA_AWS_REGION:-}
  # Paths INSIDE the container. Mount the files yourself — this file cannot guess
  # a host path.
  KAFKA_SSL_CA_LOCATION: ${KAFKA_SSL_CA_LOCATION:-}
  # mTLS, the two-path shape the Go and Python clients read.
  KAFKA_SSL_CERT_LOCATION: ${KAFKA_SSL_CERT_LOCATION:-}
  KAFKA_SSL_KEY_LOCATION: ${KAFKA_SSL_KEY_LOCATION:-}
  # The SAME keypair in the shape a JVM can load: ONE file holding chain + key.
  # Optional -- kafka-init and kafka-connect build that file from the pair above
  # when this is unset (the connect image's connect-entrypoint.sh). Set it only
  # to use a combined file you already have; it then wins over the pair. Inert
  # in the Go and Python services.
  KAFKA_SSL_KEYSTORE_LOCATION: ${KAFKA_SSL_KEYSTORE_LOCATION:-}
  # Disables certificate verification, and is logged unconditionally when set: a
  # self-signed lab, not prod.
  #
  # TWO SPELLINGS FOR ONE SETTING, and that is not tidy-uppable here. Python
  # reads only KAFKA_SSL_SKIP_VERIFY (kafka_security.py:54); Go reads both
  # (config.go:80 and the alias at :115). Shipping the Go spelling alone leaves
  # verification silently ON for the Python half -- the exact split the Go alias
  # was added to end. So both names are emitted, each defaulting to the other,
  # and an operator who sets either one gets both.
  # TWO SPELLINGS FOR ONE SETTING, and that is not tidy-uppable here. Python
  # reads only KAFKA_SSL_SKIP_VERIFY (kafka_security.py:54); Go reads both
  # (config.go:80 and the alias at :115). Shipping the Go spelling alone leaves
  # verification silently ON for the Python half -- the exact split the Go alias
  # was added to end. So both names are emitted, each defaulting to the other,
  # and an operator who sets either one gets both. Identical to the pair in
  # docker-compose.quickstart.yml; the key SETS of the two anchors must match,
  # which llm-service/tests/test_compose_kafka_security_env.py asserts.
  KAFKA_SSL_SKIP_VERIFY: ${KAFKA_SSL_SKIP_VERIFY:-${KAFKA_SSL_INSECURE_SKIP_VERIFY:-}}
  KAFKA_SSL_INSECURE_SKIP_VERIFY: ${KAFKA_SSL_INSECURE_SKIP_VERIFY:-${KAFKA_SSL_SKIP_VERIFY:-}}

# ─────────────────────────────────────────────────────────────────────────────
# Build info baked into the api-gateway, orchestrator, temporal-adapter and
# llm-service images: each serves it on GET /version, and the admin drift check
# (GET /api/v1/admin/drift) compares the commits. Compose cannot run git, so the
# caller exports these -- scripts/deploy-service.sh does, and so do the upgrade
# steps in docs/deployment/self-hosting.md. A build without them bakes in
# "dev"/"unknown", which the drift check reports as "commit unknown".
# Published images get the release sha from .github/workflows/docker-publish.yml.
# ─────────────────────────────────────────────────────────────────────────────
x-build-info: &build-info
  GIT_COMMIT: ${GIT_COMMIT:-dev}
  BUILD_TIME: ${BUILD_TIME:-unknown}

services:
  # ============================================
  # INFRASTRUCTURE
  # ============================================
  
  # NOTE: this bundle ships an OTel Collector but no telemetry backend.
  # Services export to otel-collector:4317; the collector forwards to whatever
  # OTLP endpoint OTLP_BACKEND_ENDPOINT names. Leave it at the default and
  # nothing is stored — the supported story here is `docker compose logs`.

  # Read-only Docker socket proxy. Lets api-gateway surface LIVE MCP container
  # status (docker_status=running, port), and lets the orchestrator's sentinel tell
  # a connector this host never deployed from one that is deployed and broken —
  # both without mounting the raw socket or running as root, since it can only GET
  # /containers/json (POST disabled). Wired via MCP_DOCKER_API_URL on both. Prod
  # defines its own copy for Traefik in docker-compose.prod.yml; this base copy
  # also serves api-gateway and the orchestrator locally.
  docker-socket-proxy:
    # Pinned, not `latest`. This container IS the docker-socket boundary, and `latest`
    # moved v0.4.2 -> v0.5.0 on 2026-07-27 with no change on our side; deploy-service.sh
    # runs `--force-recreate`, so that arrives on the next deploy of this service.
    # v0.5.0 is the digest `latest` resolves to today, so this pin froze it, not changed it.
    image: tecnativa/docker-socket-proxy:v0.5.0
    container_name: rsync-ai-docker-socket-proxy
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    restart: unless-stopped
    environment:
      CONTAINERS: 1
      NETWORKS: 1
      SERVICES: 1
      TASKS: 1
      VERSION: 1
      POST: 0
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
    networks:
      - default

  # PostgreSQL Database (with CDC/Logical Replication enabled)
  postgres:
    image: postgres:16-alpine
    container_name: postgres
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    ports:
      # Every host port in this file binds 127.0.0.1. A port with no host IP binds
      # 0.0.0.0, and Docker writes its own iptables rules AHEAD of ufw/firewalld,
      # so "the firewall is on" does not close it -- on a laptop that is the coffee
      # shop's Wi-Fi reaching an unauthenticated Postgres, Kafka, Redis and the
      # orchestrator's internal routes. The two user-facing ports (frontend,
      # api-gateway) read RSYNC_BIND_ADDR like docker-compose.quickstart.yml does,
      # so a dev who wants the UI on the LAN sets that one knob and exposes only
      # those two. Nothing else here is meant to be reachable off the box.
      - "127.0.0.1:${RSYNC_HP_POSTGRES:-5432}:5432"
    environment:
      - POSTGRES_USER=user
      - POSTGRES_PASSWORD=password
      - POSTGRES_DB=pipeline_db
    command: 
      - "postgres"
      - "-c"
      - "wal_level=logical"
      - "-c"
      - "max_wal_senders=1"
      - "-c"
      - "max_replication_slots=1"
    volumes:
      - postgres_data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U user -d pipeline_db"]
      interval: 5s
      timeout: 5s
      retries: 5

  # Kafka (KRaft Mode - No ZooKeeper)
  # Uses apache/kafka — Apache 2.0 licensed (replaces Confluent cp-kafka which is CCL licensed)
  # apache/kafka uses the same KAFKA_* env var convention as cp-kafka — no rename needed.
  kafka:
    image: apache/kafka:3.7.0
    hostname: kafka
    container_name: rsync-ai-kafka
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    ports:
      - "127.0.0.1:${RSYNC_HP_KAFKA:-9092}:9092"
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      # Partition counts for the two INTERNAL topics. Kafka defaults both to 50,
      # and nothing in this repo overrode them, so a single bundled broker was
      # carrying 100 internal partitions -- each one a log dir with its own
      # segment plus offset/time index, against KAFKA_HEAP_OPTS of -Xmx512m.
      # This broker serves one platform with a handful of consumer groups; 8 is
      # ample coordinator spread and cuts the internal partition count by ~84%.
      #
      # Takes effect on a FRESH cluster ONLY. __consumer_offsets and
      # __transaction_state are created once, on first use, with whatever this
      # said at the time; an existing volume keeps its 50 and ignores the
      # setting. So this buys nothing on a running deployment and everything on
      # a new self-host install -- do NOT delete the topics to "apply" it, that
      # discards every committed offset.
      KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 8
      KAFKA_TRANSACTION_STATE_LOG_NUM_PARTITIONS: 8
      # Consumer group rebalance settings (CRITICAL for multi-worker stability)
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 3000
      KAFKA_GROUP_MAX_SESSION_TIMEOUT_MS: 180000
      KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS: 6000
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_PROCESS_ROLES: 'broker,controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093'
      # Bind to 0.0.0.0 (canonical fix for i/o timeouts)
      KAFKA_LISTENERS: 'PLAINTEXT://0.0.0.0:29092,CONTROLLER://0.0.0.0:29093,PLAINTEXT_HOST://0.0.0.0:9092'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      KAFKA_LOG_DIRS: '/var/lib/kafka/data'
      # Broker cap on a single record. The 1 MB default rejects CDC change events for
      # rows with large JSON/text/blob columns (RecordTooLargeException → Debezium task
      # FAILED → restart loop). Kept in LOCKSTEP with CONNECT_PRODUCER_MAX_REQUEST_SIZE
      # (kafka-connect, producer side) and the kafka-sink-worker ReaderConfig.MaxBytes
      # (consumer side, which must exceed this). Topics inherit this broker default.
      KAFKA_MESSAGE_MAX_BYTES: 15728640
      # Retention policy for topics rsync.ai does NOT create itself.
      #
      # This is not redundant with kafka/topology.go: Debezium auto-creates the
      # rsync.cdc-* topics before the orchestrator's topology manager would, so
      # those topics carry NO per-topic overrides and take whatever the broker
      # says. Verified on the GCP host 2026-09-23 — `kafka-configs.sh --describe`
      # returns an empty dynamic config set for the live CDC topic, so the code's
      # values had never applied to it. This line is therefore the only place the
      # 7-day policy actually reaches them.
      #
      # 168h/delete matches Apache's own defaults, so this changes no behaviour
      # today. It pins it: an image whose defaults differ can no longer silently
      # re-open the unbounded-growth hole that fills the disk and stops BOTH data
      # planes. Topics rsync.ai creates explicitly still carry their own config, and
      # the one deliberate never-expire exception (the Debezium schema-history
      # topic) sets retention.ms=-1 per topic and overrides this.
      KAFKA_LOG_RETENTION_HOURS: 168
      KAFKA_LOG_CLEANUP_POLICY: 'delete'
      CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk'
      KAFKA_HEAP_OPTS: "-Xms256m -Xmx512m -XX:+ExitOnOutOfMemoryError"
    volumes:
      - kafka_data:/var/lib/kafka/data
    healthcheck:
      test: /opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1
      interval: 30s
      timeout: 15s
      retries: 5
      start_period: 60s

  # One-shot bootstrapper for the platform topics (pipeline.domain.events,
  # pii.scan.request, pii.scan.response); see scripts/kafka-init-new-topics.sh.
  # Ensures test suites that expect topics to exist are deterministic.
  kafka-init:
    image: apache/kafka:3.7.0
    container_name: rsync-ai-kafka-init
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    depends_on:
      kafka:
        condition: service_healthy
        # Other half of docker-compose.byo-kafka.yml — see that file's header:
        # profiles: without required:false = "depends on undefined service".
        required: false
    entrypoint: ["/bin/bash", "-lc"]
    # A bare `command: |` block is split into argv by Compose, so `bash -lc` only
    # ever received the first token ("export") and the container exited 0 without
    # running the script -- a green init that created nothing. The single-element
    # list keeps the whole script as one argument. Verified with
    # `docker compose config`: `command:` must render as ONE list item.
    command:
      - |
        export PATH="/opt/kafka/bin:$$PATH";
        # Same broker the orchestrator and kafka-connect use. Hardcoding
        # kafka:29092 here gave KAFKA_REPLICATION_FACTOR two meanings in one
        # file -- "RF on the customer's BYO cluster" for every other service,
        # "RF on the bundled single broker" for this one -- so the BYO operator
        # who correctly sets RF=3 is the one whose bootstrap dies with
        # InvalidReplicationFactorException against the bundled broker.
        # A BYO cluster with SASL/TLS also needs a client.properties, which
        # kafka-init-new-topics.sh now builds from the x-kafka-security env
        # merged in below and passes as --command-config.
        export KAFKA_BROKER="${KAFKA_BROKERS:-kafka:29092}";
        export PARTITIONS="3";
        bash /scripts/kafka-init-new-topics.sh
    environment:
      # The topic bootstrapper is a Kafka CLIENT too. Without these it speaks
      # PLAINTEXT to a SASL listener, which does not error -- it BLOCKS in the
      # handshake until the CLI's default timeout gives up, so the run appears
      # to hang rather than to be misconfigured. See x-kafka-security at the top
      # of this file; scripts/kafka-init-new-topics.sh turns them into the
      # --command-config the Kafka CLIs need.
      <<: *kafka-security
      # The bootstrapper must land on the same names the services subscribe to.
      # Bare `-`, matching kafkaclient.TopicPrefix(): an explicitly empty prefix
      # is the migration lever and must survive, which `:-` would overwrite.
      KAFKA_TOPIC_PREFIX: ${KAFKA_TOPIC_PREFIX-rsync.}
      # Same durability knobs the orchestrator reads, so topics created by the
      # bootstrapper and topics created at runtime agree. Passed as env rather
      # than exported in `command:` above so an operator override actually
      # reaches the script -- a hardcoded `export REPLICATION_FACTOR="1"` there
      # would win over anything set here.
      KAFKA_REPLICATION_FACTOR: ${KAFKA_REPLICATION_FACTOR:-}
      KAFKA_MIN_INSYNC_REPLICAS: ${KAFKA_MIN_INSYNC_REPLICAS:-}
    volumes:
      - ./scripts:/scripts:ro
    restart: "no"

  # Schema Registry — OPTIONAL, Apache 2.0 (Apicurio replaces Confluent cp-schema-registry)
  # Enable with: docker compose --profile schema-registry up -d
  schema-registry:
    image: apicurio/apicurio-registry:3.0.6
    hostname: schema-registry
    container_name: schema-registry
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    profiles:
      - schema-registry
    depends_on:
      kafka:
        condition: service_healthy
        # Other half of docker-compose.byo-kafka.yml — see that file's header:
        # profiles: without required:false = "depends on undefined service".
        required: false
    ports:
      - "127.0.0.1:${RSYNC_HP_SCHEMA_REGISTRY:-8085}:8080"
    environment:
      # No KAFKA_*/REGISTRY_* keys here, on purpose. Apicurio 3 reads apicurio.*-prefixed
      # config; KAFKA_BOOTSTRAP_SERVERS and REGISTRY_KAFKASQL_TOPIC (the Apicurio-2 spelling)
      # occur ZERO times in the runner jar, in all 356 shipped lib jars, and in the
      # entrypoint -- they were inert. Storage here is the image default sql/H2, NOT kafkasql,
      # so there is no broker to name.
      #
      # THE x-kafka-security ANCHOR ABOVE CANNOT SECURE THIS IMAGE, and an earlier version of
      # this comment wrongly said a kafkasql switch would need it. All 17 of its keys are as
      # inert here as the two deleted above: in both the env spelling and SmallRye's dotted
      # equivalent (34 spellings) they occur 0 times in the runner jar (2,419 entries) and
      # 0 times across all 356 lib jars (57,339 entries) -- against live controls measured in
      # the same scan (security.protocol 3 runner / 6 lib entries, sasl.mechanism 7 / 3,
      # bootstrap.servers 8 / 5). Merging it would read as secured and speak PLAINTEXT.
      #
      # What the image DOES read for a kafkasql client is declared by
      # io/apicurio/registry/storage/impl/kafkasql/KafkaSqlFactory.class in the runner jar:
      # APICURIO_KAFKASQL_SECURITY_PROTOCOL plus APICURIO_KAFKASQL_SECURITY_SASL_{ENABLED,
      # MECHANISM,CLIENT_ID,CLIENT_SECRET,TOKEN_ENDPOINT,LOGIN_CALLBACK_HANDLER_CLASS}. Image
      # defaults (application.properties): sasl.enabled=false (:22), mechanism=OAUTHBEARER
      # (:196), client-id/client-secret=sa (:157/:6), token.endpoint=http://localhost:8090
      # (:87), login.callback.handler.class=Strimzi's JaasClientOauthLoginCallbackHandler
      # (:162). Two of those properties are spelled with a HYPHEN -- `...sasl.client-id`,
      # `...sasl.client-secret` -- and the env names above are SmallRye's documented mapping
      # of dots AND dashes to `_`, so the underscore spelling is correct despite the hyphen.
      #
      # A real switch is therefore APICURIO_STORAGE_KIND=kafkasql +
      # APICURIO_KAFKASQL_BOOTSTRAP_SERVERS + that family, NOT the anchor.
      # test_compose_kafka_security_env.py requires exactly that family (and refuses to let
      # the anchor stand in for it) for any service naming the Apicurio broker key.
      #
      # NOTE: H2 here is in-memory and this service declares no volumes: -- schemas do NOT
      # survive a restart.
      QUARKUS_HTTP_PORT: 8080
    healthcheck:
      test: curl -f http://localhost:8080/health/ready || exit 1
      interval: 10s
      timeout: 5s
      retries: 5

  # ============================================
  # TEMPORAL - Workflow Orchestration
  # ============================================
  
  # Temporal Server
  temporal:
    image: temporalio/auto-setup:1.22.7
    container_name: temporal
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    depends_on:
      postgres:
        condition: service_healthy
        # Other half of docker-compose.byo-postgres.yml — see that file's header:
        # profiles: without required:false = "depends on undefined service".
        required: false
    environment:
      - DB=postgresql
      - DB_PORT=5432
      - POSTGRES_USER=user
      - POSTGRES_PWD=password
      - POSTGRES_SEEDS=postgres
      - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development-sql.yaml
    ports:
      - "127.0.0.1:${RSYNC_HP_TEMPORAL:-7233}:7233"
    healthcheck:
      test: ["CMD", "tctl", "--address", "temporal:7233", "workflow", "list"]
      interval: 10s
      timeout: 5s
      retries: 5
    volumes:
      - ./deploy/temporal:/etc/temporal/config/dynamicconfig

  # Temporal UI
  temporal-ui:
    image: temporalio/ui:2.21.3
    container_name: temporal-ui
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    depends_on:
      - temporal
    environment:
      - TEMPORAL_ADDRESS=temporal:7233
      - TEMPORAL_CORS_ORIGINS=http://localhost:3000
    ports:
      - "127.0.0.1:${RSYNC_HP_TEMPORAL_UI:-8233}:8080"

  # Temporal Admin Tools
  temporal-admin-tools:
    image: temporalio/admin-tools:1.22.7
    container_name: temporal-admin-tools
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    depends_on:
      - temporal
    environment:
      - TEMPORAL_ADDRESS=temporal:7233
    stdin_open: true
    tty: true

  # Temporal Adapter (Bridges Temporal ↔ Kafka)
  temporal-adapter:
    build:
      # Root context: the adapter `replace`s ../shared/go/kafkaclient, which a
      # ./backend-temporal-adapter context cannot reach. Matches api-gateway
      # and orchestrator, which already build from the repo root.
      context: .
      dockerfile: backend-temporal-adapter/Dockerfile
      args: *build-info
    container_name: rsync-ai-temporal-adapter
    depends_on:
      temporal:
        condition: service_healthy
      kafka:
        condition: service_healthy
        # Other half of docker-compose.byo-kafka.yml — see that file's header:
        # profiles: without required:false = "depends on undefined service".
        required: false
    environment:
      # Kafka client security for a BYO/managed broker. See x-kafka-security at
      # the top of this file; every key defaults to empty and empty means unset,
      # so the bundled broker is unaffected.
      <<: *kafka-security
      TEMPORAL_ADDRESS: "temporal:7233"
      KAFKA_BOOTSTRAP_SERVERS: "${KAFKA_BROKERS:-kafka:29092}"
      # Namespace for every Kafka topic this product creates, so an operator can
      # tell them apart from other applications' topics on a shared (BYO-Kafka)
      # cluster. Every service that touches Kafka must agree on this value: a
      # producer and a consumer resolving different names do not error, the
      # consumer just blocks on a topic nobody writes. `-` not `:-` on purpose --
      # an explicit empty value means "disable" (keep pre-prefix topic names) and
      # `:-` would silently turn that back into the default.
      KAFKA_TOPIC_PREFIX: "${KAFKA_TOPIC_PREFIX-rsync.}"
      LOG_FORMAT: "json"
      LOG_LEVEL: "info"
      ENVIRONMENT: "development"
      # SECURITY: Must match api-gateway/orchestrator ENCRYPTION_KEY for decrypting connection configs in healing activities
      ENCRYPTION_KEY: "${ENCRYPTION_KEY:-dev-encryption-key-32-bytes-long!!}"
      # Database (StateUpdateActivity / connection validation lookups)
      POSTGRES_HOST: "postgres"
      POSTGRES_PORT: "5432"
      POSTGRES_USER: "user"
      POSTGRES_PASSWORD: "password"
      POSTGRES_DB: "pipeline_db"
      # V2 Correlation Store (Phase D)
      REDIS_ADDRESS: "redis:6379"
      REDIS_PASSWORD: "${REDIS_PASSWORD:-}"
      # Scheduled saved-query model runs (RunModelActivity) call the gateway's internal
      # endpoint over authenticated S2S. Without both of these a fired schedule fails
      # closed instead of rebuilding its table.
      API_GATEWAY_URL: "http://api-gateway:8080"
      INTERNAL_SERVICE_SECRET: "${INTERNAL_SERVICE_SECRET:-}"
      # OpenTelemetry
      OTEL_EXPORTER_OTLP_ENDPOINT: "otel-collector:4317"
      OTEL_SERVICE_NAME: "temporal-adapter"
      OTEL_ENABLED: "${OTEL_ENABLED:-false}"
    ports:
      - "127.0.0.1:${RSYNC_HP_TEMPORAL_ADAPTER:-8082}:8082"
    volumes:
      # Required for connector filesystem checks + (optional) connector generation output
      - ./shared:/app/shared
    extra_hosts:
      - "host.docker.internal:host-gateway"
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"

  # ============================================
  # CDC & DATA MOVEMENT
  # ============================================

  # Kafka Connect (Debezium) — required for CDC pipelines
  kafka-connect:
    build:
      context: ./shared/internal/infra/kafka-connect
      dockerfile: Dockerfile
    container_name: kafka-connect
    # Without this the container defaults to RestartPolicy=no, and a single OOM
    # kill stops CDC capture PERMANENTLY: Debezium is the only producer into the
    # rsync.cdc-* topics, so the sink drains what is already there, every topic
    # reaches lag 0, and the pipeline still reads `running` in the control plane.
    # A dead producer and a caught-up one are indistinguishable from lag alone.
    # Measured on the demo VM 2026-09-23: 877k source rows silently uncaptured
    # until the container was started by hand. kafka-mcp-sink-mcp, the consuming
    # half of the same path, has carried this policy all along.
    restart: unless-stopped
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    depends_on:
      kafka:
        condition: service_healthy
        # Other half of docker-compose.byo-kafka.yml — see that file's header:
        # profiles: without required:false = "depends on undefined service".
        required: false
    ports:
      - "127.0.0.1:${RSYNC_HP_CONNECT:-8083}:8083"
    environment:
      # ── debezium/connect env-var convention (NOT Confluent cp-kafka-connect) ──
      # The Debezium image's entrypoint reads these UNPREFIXED names to build
      # connect-distributed.properties. The old CONNECT_* names were silently
      # ignored → "CONFIG_STORAGE_TOPIC must be set" crash.
      BOOTSTRAP_SERVERS: "${KAFKA_BROKERS:-kafka:29092}"
      GROUP_ID: "rsync-connect-cluster"
      CONFIG_STORAGE_TOPIC: "_rsync-connect-configs"
      OFFSET_STORAGE_TOPIC: "_rsync-connect-offsets"
      STATUS_STORAGE_TOPIC: "_rsync-connect-status"
      # Replication factor for Connect's three internal topics.
      #
      # These MUST carry the CONNECT_ prefix. The unprefixed spelling used here
      # before (CONFIG_STORAGE_REPLICATION_FACTOR: 1) was silently ignored: the
      # entrypoint hand-exports a FIXED list of unprefixed names (BOOTSTRAP_SERVERS,
      # GROUP_ID, the three *_STORAGE_TOPICs, the converters, the flush timers) and
      # the string "REPLICATION" appears nowhere in it, so anything else unprefixed
      # is dropped without a word. It read as working only because the value being
      # ignored (1) happened to equal the image's shipped default. Verified against
      # quay.io/debezium/connect:3.1.3.Final: with the unprefixed name set to 7 the
      # generated connect-distributed.properties still said 1.
      #
      # min.insync.replicas has no Connect worker property, so RF is the only lever
      # here -- which makes it the one that matters. On a BYO cluster whose broker
      # default is min.insync.replicas=2 (MSK, most managed clusters running a
      # pre-3.7 broker), RF=1 internal topics are created successfully and are then
      # unwritable: the worker starts, the REST API answers, and storing a connector
      # config hangs.
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "${KAFKA_REPLICATION_FACTOR:-1}"
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "${KAFKA_REPLICATION_FACTOR:-1}"
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "${KAFKA_REPLICATION_FACTOR:-1}"
      KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      OFFSET_FLUSH_INTERVAL_MS: 10000
      ADVERTISED_HOST_NAME: "kafka-connect"

      # JSON converters without embedded schemas (Debezium honors CONNECT_-prefixed
      # extras as additional worker properties).
      CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "false"
      CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
      CONNECT_PRODUCER_ACKS: "all"
      CONNECT_PRODUCER_RETRIES: 10
      # Worker-level producer cap so EVERY connector (incl. Debezium CDC) may emit records
      # for rows with large JSON/text/blob columns instead of failing the task with
      # RecordTooLargeException. Worker-level (not per-connector producer.override.*) so it
      # needs no client-config override policy. LOCKSTEP with KAFKA_MESSAGE_MAX_BYTES (broker)
      # and the kafka-sink-worker ReaderConfig.MaxBytes (consumer).
      CONNECT_PRODUCER_MAX_REQUEST_SIZE: 15728640
      CONNECT_CONSUMER_AUTO_OFFSET_RESET: "earliest"

      # ── BYO-Kafka security: SASL / TLS ───────────────────────────────────────
      # Mapping convention, read out of the pinned base image's
      # /docker-entrypoint.sh (quay.io/debezium/connect:3.1.3.Final, the
      # "Process all environment variables that start with 'CONNECT_'" loop):
      # every env var matching ^CONNECT_ becomes one connect-distributed.properties
      # line -- strip CONNECT_, lowercase, translate '_' to '.'. So
      # CONNECT_SASL_JAAS_CONFIG -> sasl.jaas.config. A variable that does NOT
      # match ^CONNECT_ is discarded in silence, which is why these carry the
      # prefix while BOOTSTRAP_SERVERS/GROUP_ID/*_STORAGE_TOPIC above deliberately
      # do not (those few are re-exported as CONNECT_* by the entrypoint itself).
      #
      # The KAFKA_* contract every other service reads (x-kafka-security at the
      # top of this file) still arrives here, because this image runs its own
      # entrypoint first (shared/internal/infra/kafka-connect/connect-entrypoint.sh,
      # the same code Helm runs) and it derives the CONNECT_* lines from it:
      #   * KAFKA_SASL_USERNAME/PASSWORD, or the KAFKA_SASL_OAUTHBEARER_* set,
      #     become the JAAS line for the mechanism;
      #   * KAFKA_SSL_CERT_LOCATION + KAFKA_SSL_KEY_LOCATION become the one-file
      #     PEM keystore a JVM needs (KAFKA_SSL_KEYSTORE_LOCATION, if set, wins).
      # Mount the certificate files into this container too, at the same paths.
      <<: *kafka-security
      #
      # Connect needs this in THREE places. The worker's own coordination/admin
      # client is a different client from the producer each source task writes
      # with and the consumer each sink task reads with, and the task-level ones
      # do NOT inherit reliably. Configure only the worker and the failure is
      # silent-partial: the worker authenticates, joins the group, serves the REST
      # API and reports connectors RUNNING, while every task fails its own
      # handshake and not one row moves.
      #
      # Unset, these render the pre-existing local behaviour exactly:
      # security.protocol=PLAINTEXT, and sasl.mechanism is inert because no SASL
      # handshake is attempted.
      CONNECT_SECURITY_PROTOCOL: "${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}"
      CONNECT_SASL_MECHANISM: "${KAFKA_SASL_MECHANISM:-PLAIN}"
      CONNECT_PRODUCER_SECURITY_PROTOCOL: "${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}"
      CONNECT_PRODUCER_SASL_MECHANISM: "${KAFKA_SASL_MECHANISM:-PLAIN}"
      CONNECT_CONSUMER_SECURITY_PROTOCOL: "${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}"
      CONNECT_CONSUMER_SASL_MECHANISM: "${KAFKA_SASL_MECHANISM:-PLAIN}"
      #
      # An OVERRIDE, normally left unset: the entrypoint builds the JAAS line from
      # KAFKA_SASL_USERNAME/PASSWORD (or the OAuth set), choosing the login module
      # the mechanism needs and escaping the values for the two grammars they
      # cross. Set KAFKA_SASL_JAAS_CONFIG only for a login module it has no arm
      # for; a non-empty value is used as-is and nothing is built.
      CONNECT_SASL_JAAS_CONFIG: "${KAFKA_SASL_JAAS_CONFIG:-}"
      CONNECT_PRODUCER_SASL_JAAS_CONFIG: "${KAFKA_SASL_JAAS_CONFIG:-}"
      CONNECT_CONSUMER_SASL_JAAS_CONFIG: "${KAFKA_SASL_JAAS_CONFIG:-}"
      #
      # Truststore PATH -- and only the path -- is a bare pass-through key with no
      # `${...:-}` default. A key with no value is OMITTED from the container
      # environment when the host variable is unset, whereas `${VAR:-}` renders it
      # present-but-empty, and the debezium entrypoint writes every CONNECT_ var it
      # sees, empty included. Kafka then reads an empty ssl.truststore.location as a
      # real path: "Failed to load PEM SSL keystore" / NoSuchFileException, worker
      # exit 2 (reproduced against this image). An empty default would therefore
      # break the commonest BYO case outright -- Confluent Cloud and MSK present
      # certificates chaining to a public CA that is already in the JVM default
      # truststore, so they need NO truststore setting at all.
      #
      # Compose cannot omit a key conditionally on a DIFFERENTLY named variable, so
      # the path keeps the CONNECT_ names themselves, and all THREE must be set.
      # Setting only the worker one is the classic Connect partial failure: the
      # worker starts and the REST API answers, while source-task producers and
      # sink-task consumers fall back to the default truststore and fail their own
      # handshake later, far from the config that caused it.
      #
      #   export CONNECT_SSL_TRUSTSTORE_LOCATION=/connect-certs/ca.pem
      #   export CONNECT_PRODUCER_SSL_TRUSTSTORE_LOCATION="$CONNECT_SSL_TRUSTSTORE_LOCATION"
      #   export CONNECT_CONSUMER_SSL_TRUSTSTORE_LOCATION="$CONNECT_SSL_TRUSTSTORE_LOCATION"
      #   export KAFKA_SSL_TRUSTSTORE_TYPE=PEM   # one variable, reaches all three
      #
      CONNECT_SSL_TRUSTSTORE_LOCATION:
      CONNECT_PRODUCER_SSL_TRUSTSTORE_LOCATION:
      CONNECT_CONSUMER_SSL_TRUSTSTORE_LOCATION:
      #
      # The PASSWORD is pass-through for the same reason, and the reason is sharper
      # than it looks. Kafka rejects a truststore password outright for PEM stores:
      #
      #   InvalidConfigurationException: SSL trust store password cannot be
      #   specified for PEM format.
      #
      # and an EMPTY string counts as specified (ConfigDef turns "" into a
      # non-null Password). So `${KAFKA_SSL_TRUSTSTORE_PASSWORD:-}` would kill the
      # worker at boot for every operator using a PEM CA bundle -- the modern,
      # password-less form -- while blaming a password they never set. Verified
      # against this image: PEM + empty password exits 2, PEM + absent password
      # starts and reaches the broker.
      CONNECT_SSL_TRUSTSTORE_PASSWORD:
      CONNECT_PRODUCER_SSL_TRUSTSTORE_PASSWORD:
      CONNECT_CONSUMER_SSL_TRUSTSTORE_PASSWORD:
      # The TYPE is safe to default and so takes one variable for all three
      # clients: it is consulted only when a location is set, and JKS is Kafka's
      # own default, so an always-present ssl.truststore.type=JKS changes nothing.
      CONNECT_SSL_TRUSTSTORE_TYPE: "${KAFKA_SSL_TRUSTSTORE_TYPE:-JKS}"
      CONNECT_PRODUCER_SSL_TRUSTSTORE_TYPE: "${KAFKA_SSL_TRUSTSTORE_TYPE:-JKS}"
      CONNECT_CONSUMER_SSL_TRUSTSTORE_TYPE: "${KAFKA_SSL_TRUSTSTORE_TYPE:-JKS}"
      # Hostname verification. Bare `-`, not `:-`: unset means Kafka's default
      # ("https", verify), while an explicitly EMPTY value is Kafka's documented off
      # switch and has to survive as empty -- `:-` would substitute the default back
      # over that deliberate choice. Same idiom as KAFKA_TOPIC_PREFIX.
      CONNECT_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "${KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM-https}"
      CONNECT_PRODUCER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "${KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM-https}"
      CONNECT_CONSUMER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "${KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM-https}"

      # ── FileConfigProvider ──
      # Resolve ${file:/connect-secrets/<name>.properties:key} references at task
      # start so source DB passwords never enter the Kafka config topic nor the
      # GET /connectors/<name>/config REST response. debezium-mcp writes the
      # per-connector secret file to the shared connect_secrets volume; this worker
      # reads it (ro). allowed.paths confines the provider to the secrets dir
      # (Kafka >= 3.5; debezium/connect:3.0 ships Kafka 3.8).
      CONNECT_CONFIG_PROVIDERS: "file"
      CONNECT_CONFIG_PROVIDERS_FILE_CLASS: "org.apache.kafka.common.config.provider.FileConfigProvider"
      CONNECT_CONFIG_PROVIDERS_FILE_PARAM_ALLOWED_PATHS: "/connect-secrets"

      # JVM heap (avoid OOM exit 137 on dev machines)
      KAFKA_HEAP_OPTS: "-Xms256M -Xmx768M -XX:+ExitOnOutOfMemoryError"

    volumes:
      # Read-only: the worker only RESOLVES secrets that debezium-mcp writes.
      - connect_secrets:/connect-secrets:ro

    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8083/"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s

  # ============================================
  # INTERNAL MCP SERVICES (CORE / SELF-HOSTED)
  # ============================================
  # These are platform/internal connectors that run in the core `rsync-ai` project.
  # Only user/external MCP connectors should run in the separate `rsync-ai-mcp` project.

  debezium-mcp:
    build:
      context: ./shared/mcp-connectors/internal/debezium/versions/v1.0.0
      dockerfile: Dockerfile
    container_name: rsync-ai-debezium-v1-0-0-mcp
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    labels:
      - rsync-ai.mcp=true
      - rsync-ai.connector=debezium
    environment:
      # Kafka client credentials. This connector configures Debezium's *schema
      # history* producer AND consumer, which are separate Kafka clients from
      # the Connect worker's own — it reads KAFKA_SECURITY_PROTOCOL /
      # KAFKA_SASL_* / KAFKA_SSL_* straight out of this env
      # (versions/v1.0.0/test_schema_history_security.py proves each name).
      # The Dockerfile's KAFKA_BOOTSTRAP_SERVERS default supplies only an
      # ADDRESS, so without the anchor a BYO broker is dialled anonymously over
      # PLAINTEXT and the failure reads as an ordinary broker error rather than
      # a missing-config one. Address and credentials must travel together: a
      # correct key aimed at the bundled broker is the same outage with a more
      # confusing message, which is why KAFKA_BROKERS is declared right here
      # and not left to the image default.
      <<: *kafka-security
      KAFKA_BROKERS: ${KAFKA_BROKERS:-kafka:29092}
      PORT: "8000"
      MCP_PORT: "8000"
      MCP_HTTP_MODE: "true"
      DOCKER_CONTAINER: "true"
      LOG_FORMAT: json
      LOG_LEVEL: info
      ENVIRONMENT: development
      # Externalize per-connector secrets to this shared-with-kafka-connect volume so
      # the source password stays OUT of the Kafka config topic. Unset ⇒ inline config
      # (backward compatible). See connector.py externalize_secrets().
      DEBEZIUM_SECRETS_DIR: /connect-secrets
    restart: unless-stopped
    volumes:
      # Read-write: debezium-mcp WRITES the secret .properties the worker resolves.
      - connect_secrets:/connect-secrets
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      start_period: 10s
      retries: 3

  # MinIO S3-compatible object storage backend.
  # Required for claim-check staging when batch payloads exceed the inline Kafka size cap.
  # Without this, minio-mcp and kafka-mcp-sink fall back to chunked inline Kafka writes.
  minio:
    # Pinned: an unpinned object store silently changes under a re-pull, and this one
    # holds staged pipeline data. Pinned by digest because cgr.dev/chainguard serves
    # only `latest` (and `latest-dev`) anonymously, so a digest is the only pin there is.
    # Chainguard's build, not MinIO's: MinIO withdrew anonymous pulls from
    # docker.io/minio/* (2026-09-12) and then from quay.io/minio/* (401). One multi-arch
    # image carries both `minio` (its entrypoint) and `mc`, so the healthcheck below
    # and minio-lifecycle-init use it too. It has no curl: health checks go through mc.
    image: cgr.dev/chainguard/minio@sha256:bd014394a80898e68c149f2311fdf8d5a2c2f3bb2c33b9327ae6d02b4b065ae1
    # Root, like the image this replaces. A volume written by v0.1.5 is root-owned
    # 0755, and this image's default user (65532) cannot write it: the server exits
    # "Unable to write to the backend" and restarts forever, and everything that
    # waits on its health never starts.
    user: "0:0"
    container_name: rsync-ai-minio
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    restart: unless-stopped
    environment:
      MINIO_ROOT_USER: ${MINIO_ACCESS_KEY_ID:-minioadmin}
      MINIO_ROOT_PASSWORD: ${MINIO_SECRET_ACCESS_KEY:-minioadmin}
    command: server /data --console-address ":9001"
    volumes:
      - minio_data:/data
    healthcheck:
      test: ["CMD", "mc", "ready", "local"]
      interval: 10s
      timeout: 5s
      retries: 5

  # One-shot: ensure the claim-check bucket exists and apply an expiration
  # lifecycle rule on the staging prefix. This is the BACKSTOP for MinIO disk
  # growth — the sink deletes each claim-check object after a successful write,
  # but if a delete is missed (worker crash, etc.) this rule reclaims the object
  # after MINIO_STAGING_EXPIRE_DAYS.
  #
  # TTL MUST be >= the Kafka batch-topic retention (7 days, see
  # backend-orchestrator/internal/kafka/topology.go retention.ms). For a >256KB
  # batch only a pointer goes to Kafka while the rows live as a MinIO object; if
  # the staged object expired BEFORE its Kafka pointer, a lagging sink would read
  # NoSuchKey and lose those rows. With uncapped chunked transfers a sink can lag
  # for hours/days on a multi-million-row backfill, so the old 2-day default was
  # SHORTER than retention and opened a loss window — default raised to 7 days.
  minio-lifecycle-init:
    # The server image above, same digest, for the same reasons; mc writes the
    # lifecycle rule. Chainguard's mc-only image is distroless (no /bin/sh for the
    # entrypoint below), so the server image, which ships both, serves here too.
    # No `user:` needed: nothing here writes the data volume.
    image: cgr.dev/chainguard/minio@sha256:bd014394a80898e68c149f2311fdf8d5a2c2f3bb2c33b9327ae6d02b4b065ae1
    container_name: rsync-ai-minio-lifecycle-init
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    depends_on:
      minio:
        condition: service_healthy
    environment:
      - MINIO_BUCKET=${MINIO_BUCKET:-pipeline-data}
      - MINIO_PREFIX=${MINIO_PREFIX:-staging}
      - MINIO_ACCESS_KEY_ID=${MINIO_ACCESS_KEY_ID:-minioadmin}
      - MINIO_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minioadmin}
      - MINIO_STAGING_EXPIRE_DAYS=${MINIO_STAGING_EXPIRE_DAYS:-7}
    entrypoint: ["/bin/sh", "-c"]
    command:
      - |
        set -e
        until mc alias set local http://minio:9000 "$$MINIO_ACCESS_KEY_ID" "$$MINIO_SECRET_ACCESS_KEY"; do
          echo "waiting for minio..."; sleep 2;
        done
        mc mb --ignore-existing "local/$$MINIO_BUCKET"
        # Apply expiration on the staging prefix (newer mc: `ilm rule add`, older: `ilm add`)
        mc ilm rule add "local/$$MINIO_BUCKET" --expire-days "$$MINIO_STAGING_EXPIRE_DAYS" --prefix "$$MINIO_PREFIX/" \
          || mc ilm add --expiry-days "$$MINIO_STAGING_EXPIRE_DAYS" --prefix "$$MINIO_PREFIX/" "local/$$MINIO_BUCKET" \
          || echo "WARN: could not set ILM rule (mc version?); sink-side delete is primary cleanup"
        echo "✅ minio-init: bucket + lifecycle ready"
    restart: "no"

  # The blob lane's staging store: a second MinIO on the connector network ONLY.
  # Blob passthrough has the SOURCE connector stage each object here and the
  # DESTINATION connector fetch it, and connectors join only rsync-ai-mcp
  # (SEC-M-06). `minio` above is on `default` only, so a connector cannot resolve
  # it, and bridging it would hand every connector the store that holds every
  # pipeline's staged claim-check rows. This one holds only staged blobs, under
  # its own login. Keep ":9000" in its URL: the connectors' staging client uses
  # path-style addressing only for an endpoint with "minio", "localhost" or
  # ":9000" in it (base_connector.py _get_staging_client).
  # KI-BLOB-LANE-STAGING-UNREACHABLE-FROM-CONNECTORS.
  blob-staging:
    image: cgr.dev/chainguard/minio@sha256:bd014394a80898e68c149f2311fdf8d5a2c2f3bb2c33b9327ae6d02b4b065ae1
    container_name: ${STACK_PREFIX:-rsync-ai}-blob-staging
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    restart: unless-stopped
    environment:
      MINIO_ROOT_USER: ${BLOB_STAGING_ACCESS_KEY:-${MINIO_ACCESS_KEY_ID:-minioadmin}}
      MINIO_ROOT_PASSWORD: ${BLOB_STAGING_SECRET_KEY:-${MINIO_SECRET_ACCESS_KEY:-minioadmin}}
    command: server /data
    volumes:
      - blob_staging_data:/data
    networks:
      - rsync-ai-mcp
    healthcheck:
      test: ["CMD", "mc", "ready", "local"]
      interval: 10s
      timeout: 5s
      retries: 5

  # Bucket + expiry for blob-staging. Nothing deletes a staged blob per message:
  # blobs are content-addressed and shared between messages, so this rule is the
  # only cleanup. Same TTL floor as minio-lifecycle-init, for the same reason.
  blob-staging-init:
    image: cgr.dev/chainguard/minio@sha256:bd014394a80898e68c149f2311fdf8d5a2c2f3bb2c33b9327ae6d02b4b065ae1
    container_name: ${STACK_PREFIX:-rsync-ai}-blob-staging-init
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    depends_on:
      blob-staging:
        condition: service_healthy
    environment:
      - MINIO_BUCKET=${MINIO_BUCKET:-pipeline-data}
      - MINIO_PREFIX=${MINIO_PREFIX:-staging}
      - BLOB_STAGING_ACCESS_KEY=${BLOB_STAGING_ACCESS_KEY:-${MINIO_ACCESS_KEY_ID:-minioadmin}}
      - BLOB_STAGING_SECRET_KEY=${BLOB_STAGING_SECRET_KEY:-${MINIO_SECRET_ACCESS_KEY:-minioadmin}}
      - MINIO_STAGING_EXPIRE_DAYS=${MINIO_STAGING_EXPIRE_DAYS:-7}
    networks:
      - rsync-ai-mcp
    entrypoint: ["/bin/sh", "-c"]
    command:
      - |
        set -e
        until mc alias set local http://blob-staging:9000 "$$BLOB_STAGING_ACCESS_KEY" "$$BLOB_STAGING_SECRET_KEY"; do
          echo "waiting for blob-staging..."; sleep 2;
        done
        mc mb --ignore-existing "local/$$MINIO_BUCKET"
        mc ilm rule add "local/$$MINIO_BUCKET" --expire-days "$$MINIO_STAGING_EXPIRE_DAYS" --prefix "$$MINIO_PREFIX/" \
          || mc ilm add --expiry-days "$$MINIO_STAGING_EXPIRE_DAYS" --prefix "$$MINIO_PREFIX/" "local/$$MINIO_BUCKET" \
          || echo "WARN: could not set the blob staging expiry rule"
        echo "blob-staging-init: bucket + lifecycle ready"
    restart: "no"

  # Internal: MinIO staging MCP (claim-check upload/read/delete)
  minio-mcp:
    build:
      context: ./shared/mcp-connectors/internal/minio/versions/v1.0.0
      dockerfile: Dockerfile
    container_name: rsync-ai-minio-v1-0-0-mcp
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    labels:
      - rsync-ai.mcp=true
      - rsync-ai.connector=minio
      - rsync-ai.connector_version=v1.0.0
    environment:
      - PORT=8000
      - MCP_PORT=8000
      - MCP_HTTP_MODE=true
      - DOCKER_CONTAINER=true
      - LOG_FORMAT=json
      - LOG_LEVEL=info
      - ENVIRONMENT=development
      # Optional defaults (can be overridden per request via params.config)
      # Pin to the unambiguous container_name, NOT the bare `minio` alias: the
      # e2e overlay (docker-compose.e2e.dbs.yml) ships a second container that
      # also aliases `minio` on the shared network, so `http://minio:9000`
      # DNS-round-robins between two MinIO stores. Writes landing on one and
      # reads hitting the other → NoSuchKey → DLQ → silent claim-check drop.
      # `rsync-ai-minio` resolves to exactly one container. Override via env for
      # external object stores.
      - MINIO_ENDPOINT_URL=${MINIO_ENDPOINT_URL:-http://rsync-ai-minio:9000}
      - MINIO_ACCESS_KEY_ID=${MINIO_ACCESS_KEY_ID:-minioadmin}
      - MINIO_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minioadmin}
      - MINIO_REGION=${MINIO_REGION:-us-east-1}
      - MINIO_BUCKET=${MINIO_BUCKET:-pipeline-data}
      - MINIO_PREFIX=${MINIO_PREFIX:-staging}
      # Same class of silent disagreement as RSYNC_LLM_DIAGNOSER_ENABLED above:
      # connector.py:36 reads RSYNC_CLAIM_CHECK_GZIP, and no compose file passed
      # it through, so an operator who set it in .env got plain JSON and no error
      # said why. Default stays empty (= off), so wiring it changes nothing until
      # someone sets it deliberately. The reader is flag-INDEPENDENT magic-byte
      # autodetect, so turning it on mid-flight still reads back legacy plain
      # objects -- it is safe to flip without draining staging.
      - RSYNC_CLAIM_CHECK_GZIP=${RSYNC_CLAIM_CHECK_GZIP:-}
    depends_on:
      minio:
        condition: service_healthy
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      start_period: 10s
      retries: 3

  kafka-mcp-sink-mcp:
    build:
      # Use repo root as build context so the worker can import shared/go/transforms via replace.
      context: .
      dockerfile: shared/mcp-connectors/internal/kafka-mcp-sink/Dockerfile
    container_name: rsync-ai-kafka-mcp-sink-v1-0-0-mcp
    labels:
      - rsync-ai.mcp=true
      - rsync-ai.connector=kafka-mcp-sink
    environment:
      # Kafka client credentials. The sink worker is a consumer group member on
      # every pipeline topic — initKafkaSecurity() in
      # worker-src/cmd/kafka-sink-worker/kafka_security.go reads these names,
      # and kafka_security_test.go pins them. Without the anchor a BYO broker is
      # dialled anonymously over PLAINTEXT. KAFKA_BROKERS is declared alongside
      # so credentials can never be aimed at a cluster other than the one they
      # belong to.
      <<: *kafka-security
      KAFKA_BROKERS: ${KAFKA_BROKERS:-kafka:29092}
      PORT: "8000"
      # How many destination flushes this sink runs at once. Unset/empty = 4.
      #
      # A CDC data topic is born with min(3, brokers) partitions and Debezium gives
      # every source table a topic of its own, so one worker is assigned tens of
      # partitions. It consumed all of them already, but drained them one at a time
      # because the destination write ran inline on the consume goroutine — one slow
      # table stalled every other partition. Lanes are sharded on (topic, partition),
      # so per-row order is untouched.
      #
      # Set to 0 or 1 to restore the old inline behaviour exactly; this is the kill
      # switch. Raising it opens more concurrent connections to the DESTINATION, which
      # is somebody else's database — 4 is deliberately modest.
      RSYNC_SINK_FLUSH_LANES: ${RSYNC_SINK_FLUSH_LANES:-}
      # Mirror of the orchestrator's schema-drift switch. The sink worker reports the
      # columns CDC adds on its own to rsync.healer.schema-changes only when this is
      # "true" (newSchemaDriftWriter); off, it writes nothing there, because the
      # orchestrator then neither creates nor consumes that topic. Must stay
      # byte-identical to the orchestrator line; a census test pins it.
      RSYNC_SCHEMA_DRIFT_ENABLED: "${RSYNC_SCHEMA_DRIFT_ENABLED:-false}"
      MCP_PORT: "8000"
      MCP_HTTP_MODE: "true"
      DOCKER_CONTAINER: "true"
      LOG_FORMAT: json
      LOG_LEVEL: info
      ENVIRONMENT: development
      # Allow kafka-mcp-sink to fetch claim-check objects from MinIO (S3-compatible)
      # Pin to the unambiguous container_name, NOT the bare `minio` alias: the
      # e2e overlay (docker-compose.e2e.dbs.yml) ships a second container that
      # also aliases `minio` on the shared network, so `http://minio:9000`
      # DNS-round-robins between two MinIO stores. Writes landing on one and
      # reads hitting the other → NoSuchKey → DLQ → silent claim-check drop.
      # `rsync-ai-minio` resolves to exactly one container. Override via env for
      # external object stores.
      MINIO_ENDPOINT_URL: ${MINIO_ENDPOINT_URL:-http://rsync-ai-minio:9000}
      MINIO_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY_ID:-minioadmin}
      MINIO_SECRET_ACCESS_KEY: ${MINIO_SECRET_ACCESS_KEY:-minioadmin}
      MINIO_REGION: ${MINIO_REGION:-us-east-1}
      MINIO_BUCKET: ${MINIO_BUCKET:-pipeline-data}
      MINIO_PREFIX: ${MINIO_PREFIX:-staging}
      # Sink writes the durable ACK ledger (pipeline_batch_acks). Without
      # this URL pgDB was nil and the ledger stayed empty — the postflight
      # silent-drop invariant then had to fall back to sink-emitted stats
      # events, which is a weaker signal. With it, Postgres is the single
      # source of truth for "has this Kafka offset been written to dest?".
      # URL is built from the same env vars the `postgres` service uses, so
      # changing creds in one place updates every consumer.
      POSTGRES_USER: ${POSTGRES_USER:-user}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password}
      POSTGRES_DB: ${POSTGRES_DB:-pipeline_db}
      POSTGRES_HOST: ${POSTGRES_HOST:-postgres}
      POSTGRES_PORT: "${POSTGRES_PORT:-5432}"
      POSTGRES_URL: "postgres://${POSTGRES_USER:-user}:${POSTGRES_PASSWORD:-password}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-pipeline_db}?sslmode=${DB_SSLMODE:-disable}"
      # Redis for the ACK ledger / offset checkpointing. Must carry the password
      # when Redis runs with --requirepass (staging and prod) — otherwise the
      # sink worker crashes on every start with "NOAUTH Authentication required".
      REDIS_URL: "redis://:${REDIS_PASSWORD:-}@redis:6379/0"
    # Sink worker calls destination MCPs (e.g. rsync-ai-postgresql-v1-0-0-mcp)
    # by container name. JIT-deployed MCPs live on rsync-ai-mcp; without
    # bridging into that network the sink resolves nothing and every batch
    # write fails with "no such host". This is the silent failure mode that
    # let pipelines report `completed` while leaving the destination empty.
    networks:
      - default
      - rsync-ai-mcp
    restart: unless-stopped
    # Ship the sink worker's (now structured-JSON) stdout/stderr to fluent-bit so
    # the destination-write leg is visible + trace-correlated downstream. Without
    # this driver the sink is the one data-plane stage absent from the flow view. F-Obs-2.
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      start_period: 10s
      retries: 3

  # Context7 MCP Service (Documentation Lookup)
  context7-mcp:
    build:
      context: ./shared/mcp-connectors/internal/context7
      dockerfile: Dockerfile
    container_name: rsync-ai-context7-v1-0-0-mcp
    environment:
      - PORT=8080
      # Use context7.com base URL - api. subdomain doesn't resolve
      - CONTEXT7_API_URL=https://context7.com/api
      # Provide via your shell environment or env file (do not commit keys).
      - CONTEXT7_API_KEY=${CONTEXT7_API_KEY:-}
      - CONTEXT7_DEFAULT_TOKENS=10000
      - CONTEXT7_TIMEOUT=30
      - LOG_FORMAT=json
      - LOG_LEVEL=info
      - ENVIRONMENT=development
      # OpenTelemetry Configuration
      - OTEL_EXPORTER_OTLP_ENDPOINT=otel-collector:4317
      - OTEL_SERVICE_NAME=context7-mcp
      - OTEL_ENABLED=true
    ports:
      - "127.0.0.1:${RSYNC_HP_CONTEXT7:-8087}:8080"
    # Add DNS servers to resolve external hostnames
    dns:
      - 8.8.8.8
      - 8.8.4.4
      - 1.1.1.1
    healthcheck:
      test: ["CMD", "python", "-c", "import httpx; httpx.get('http://localhost:8080/health', timeout=2)"]
      interval: 10s
      timeout: 5s
      retries: 3
    extra_hosts:
      - "host.docker.internal:host-gateway"
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    restart: unless-stopped

  # Redis (State Store for Orchestrator & Caching)
  redis:
    image: redis:7-alpine
    container_name: redis
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    ports:
      - "127.0.0.1:${RSYNC_HP_REDIS:-6379}:6379"
    command:
      - sh
      - -c
      # The password is written to a 0600 file and never appears on the
      # redis-server ARGV, so it stays out of `ps` on the host and out of
      # the rendered compose spec (`docker inspect` / `compose config`).
      - |
        umask 077
        : > /tmp/redis.conf
        if [ -n "$$REDIS_PASSWORD" ]; then
          printf 'requirepass %s\n' "$$REDIS_PASSWORD" > /tmp/redis.conf
        fi
        exec redis-server /tmp/redis.conf --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru
    environment:
      - REDIS_PASSWORD=${REDIS_PASSWORD:-}
    volumes:
      - redis_data:/data
    healthcheck:
      # REDISCLI_AUTH is read from the environment by redis-cli, so the password
      # never reaches ARGV (a healthcheck re-runs every 5s, once per interval).
      test: ["CMD-SHELL", "if [ -n \"$$REDIS_PASSWORD\" ]; then REDISCLI_AUTH=\"$$REDIS_PASSWORD\" redis-cli --no-auth-warning ping; else redis-cli ping; fi"]
      interval: 5s
      timeout: 3s
      retries: 5

  # Backend Orchestrator (Go)
  orchestrator:
    build:
      context: .
      dockerfile: backend-orchestrator/Dockerfile
      args: *build-info
    container_name: rsync-ai-orchestrator
    restart: unless-stopped
    # How long `docker compose stop/down/restart` waits after SIGTERM before it
    # SIGKILLs, which skips every deferred Stop/Close in main.go. Left unset,
    # the daemon decides: 10 s on Docker Engine, 1 s on the Docker Desktop
    # (engine 28.5.1) this was measured on. Shutdown drains HTTP for up to
    # 10 s, then gives the Kafka consumer groups up to 15 s to leave
    # (consumerCloseTimeout in backend-orchestrator/internal/kafka/manager.go),
    # on top of the agents' own Stop calls; an idle dev stack takes 3-9 s.
    # backend-orchestrator/internal/kafka/stop_grace_test.go pins this value
    # and its copies in docker-compose.quickstart.yml and the Helm chart.
    stop_grace_period: 45s
    deploy:
      # SCALING:
      #   One replica. Static membership is OPT-IN via
      #   KAFKA_STATIC_MEMBERSHIP=true and derives instance_id from $HOSTNAME
      #   (unique per pod); it only reduces rebalance churn on restarts.
      replicas: 1
    ports:
      - "127.0.0.1:${RSYNC_HP_ORCHESTRATOR:-8081}:8080" # Internal port for debugging
    environment:
      # Kafka client security for a BYO/managed broker. See x-kafka-security at
      # the top of this file; every key defaults to empty and empty means unset,
      # so the bundled broker is unaffected.
      <<: *kafka-security
      DB_HOST: "postgres"
      DB_PORT: "5432"
      DB_USER: "user"
      DB_PASSWORD: "password"
      DB_NAME: "pipeline_db"
      KAFKA_BROKERS: "${KAFKA_BROKERS:-kafka:29092}"
      # Durability for every topic the orchestrator creates. Read by
      # internal/kafka/replication.go (EnvReplicationFactor / EnvMinInsyncReplicas);
      # empty means "operator stated nothing", which is not the same as zero -- the
      # built-in derivation (1 on a single broker, min(3, brokers) beyond) applies.
      #
      # These two travel together on purpose. min.insync.replicas > replication
      # factor produces a topic that is CREATED, listed and subscribable, and then
      # rejects every acks=all produce with NOT_ENOUGH_REPLICAS: a pipeline that
      # reports running and streams zero rows. The Go side clamps misr down to the
      # topic's final RF for exactly that reason, so setting only one of these
      # cannot wedge a cluster.
      #
      # Version boundary, measured rather than assumed: on the bundled
      # apache/kafka:3.7.0 broker an RF=1 topic carrying min.insync.replicas=2
      # still ACCEPTS acks=all -- 3.7 caps the effective value at the replica count
      # (kafka.cluster.Partition#effectiveMinIsr, present in that broker's
      # kafka_2.13-3.7.0.jar). The trap is therefore real for the OLDER brokers a
      # BYO/self-host deployment actually points at (MSK and most managed clusters
      # still run 2.8-3.6), which is exactly the case these variables exist for.
      # Clamp regardless: on a 3.7+ broker the clamp is a no-op.
      KAFKA_REPLICATION_FACTOR: "${KAFKA_REPLICATION_FACTOR:-}"
      KAFKA_MIN_INSYNC_REPLICAS: "${KAFKA_MIN_INSYNC_REPLICAS:-}"
      # Partition count for every CDC data topic (backend-orchestrator/internal/kafka
      # partitions.go EnvCDCTopicPartitions). Empty means "operator stated nothing", so
      # the derivation applies: 1 on a single broker, min(3, brokers) beyond that. Set
      # it only to go WIDER than 3 on a large cluster.
      #
      # It is a birth setting, not a live one. Kafka cannot reduce a partition count at
      # all, and RAISING one on a topic that is already streaming rehashes every key
      # onto a different partition and breaks per-key ordering, so rsync.ai never resizes
      # an existing topic -- a change here reaches only topics created after it.
      KAFKA_CDC_TOPIC_PARTITIONS: "${KAFKA_CDC_TOPIC_PARTITIONS:-}"
      # Namespace for every Kafka topic this product creates, so an operator can
      # tell them apart from other applications' topics on a shared (BYO-Kafka)
      # cluster. Every service that touches Kafka must agree on this value: a
      # producer and a consumer resolving different names do not error, the
      # consumer just blocks on a topic nobody writes. `-` not `:-` on purpose --
      # an explicit empty value means "disable" (keep pre-prefix topic names) and
      # `:-` would silently turn that back into the default.
      KAFKA_TOPIC_PREFIX: "${KAFKA_TOPIC_PREFIX-rsync.}"
      # Blob-passthrough staging (universal-blob-passthrough plan §3): the
      # executor's blob lane hands these to the source connector (to stage object
      # bytes) and, via the sink message, to the destination connector (to fetch
      # them). blob-staging, NOT minio: connectors join only rsync-ai-mcp and
      # cannot resolve minio (KI-BLOB-LANE-STAGING-UNREACHABLE-FROM-CONNECTORS).
      # The lane reads no other MinIO credential, so the orchestrator holds none.
      # MINIO_BUCKET/PREFIX stay: the staged blob's bucket must be in the sink's
      # allow-list and its prefix under blob-staging-init's expiry rule.
      BLOB_STAGING_ENDPOINT_URL: "${BLOB_STAGING_ENDPOINT_URL:-http://blob-staging:9000}"
      BLOB_STAGING_ACCESS_KEY_ID: "${BLOB_STAGING_ACCESS_KEY:-${MINIO_ACCESS_KEY_ID:-minioadmin}}"
      BLOB_STAGING_SECRET_ACCESS_KEY: "${BLOB_STAGING_SECRET_KEY:-${MINIO_SECRET_ACCESS_KEY:-minioadmin}}"
      MINIO_REGION: "${MINIO_REGION:-us-east-1}"
      MINIO_BUCKET: "${MINIO_BUCKET:-pipeline-data}"
      MINIO_PREFIX: "${MINIO_PREFIX:-staging}"
      JWT_SECRET: "dev_secret_key_change_in_prod"
      # S2S secret the gateway's internal routes check (namespace lock, heal,
      # OAuth refresh) and requirePrincipal accepts. Same value as api-gateway, or
      # every one of those calls is refused 401/503 and only logged as a warning.
      INTERNAL_SERVICE_SECRET: "${INTERNAL_SERVICE_SECRET:-}"
      # Guarded operator endpoint for recovering FAILED Debezium connectors.
      # Default off for safety. Enable explicitly with CDC_RECOVERY_ENABLED=true.
      CDC_RECOVERY_ENABLED: "${CDC_RECOVERY_ENABLED:-false}"
      # Autonomous CDC sink self-heal (Bug B, #678): bounded per-pipeline sink-worker
      # auto-restart on a GENUINE 3-signal wedge (corrected lag + stale + flat). Default
      # OFF everywhere (operational-safety gate mirroring CDC_RECOVERY_ENABLED, NOT an
      # edition gate — preserves OSS/cloud parity). Arm by setting the env var to true.
      CDC_SINK_AUTORESTART_ENABLED: "${CDC_SINK_AUTORESTART_ENABLED:-false}"
      # Orphan sink-worker reaper (KI-CDC-SINK-WORKER-NO-REAPER): off | dry_run | enforce.
      # Default OFF, same op-safety gate; stops only workers whose pipeline is deleted or stopped.
      SINK_WORKER_REAPER_MODE: "${SINK_WORKER_REAPER_MODE:-off}"
      # Self-healing schema-drift detector (P0/P1): the drift-detect -> approve loop.
      # Default OFF. Flip to true to activate the executor's baseline+diff+emit
      # (schema_drift.go) AND the healer's schema-change consumer (StartSchemaOnly in
      # workers/executor.go). When false, both paths are inert no-ops -> zero behavior change.
      RSYNC_SCHEMA_DRIFT_ENABLED: "${RSYNC_SCHEMA_DRIFT_ENABLED:-false}"
      # Additive auto-apply, a SEPARATE opt-in from the line above (healer.go
      # autoApplyEnabled()). It was read by the orchestrator and documented in
      # .env.prod.example + docs/design/self-healing-drift-approve.md, but no compose
      # file passed it through — so setting it in an env file did nothing and the
      # operator had no way to tell. That silent disagreement is the bug; the default
      # stays `false`, so wiring it changes nothing until someone sets it deliberately.
      # Destructive classes ignore it entirely (DROP/TRUNCATE always need approval).
      RSYNC_SCHEMA_DRIFT_AUTOAPPLY: "${RSYNC_SCHEMA_DRIFT_AUTOAPPLY:-false}"
      # LLM error diagnoser, chained BEHIND the rules (pkg/diagnose/llm.go
      # NewFromEnv). The RuleBasedDiagnoser always runs first; the LLM is asked
      # only about a CategoryUnknown result, and its confidence is clamped below
      # heal's auto-execute band, so it can suggest but never auto-act.
      #
      # Exactly the same silent disagreement as the line above: the code read
      # RSYNC_LLM_DIAGNOSER_ENABLED and NO compose file passed it through, so an
      # operator who set it in .env got nothing and no error said why. The
      # default stays `false`, so wiring it changes nothing until someone sets
      # it deliberately — but turning it ON sends llmscrub-scrubbed failure text
      # over the network to the LLM service, which is a privacy-surface change,
      # not a config tidy-up.
      RSYNC_LLM_DIAGNOSER_ENABLED: "${RSYNC_LLM_DIAGNOSER_ENABLED:-false}"
      # Optional overrides, both empty by default (empty = fall back in code).
      # Model: RSYNC_LLM_DIAGNOSER_MODEL -> LLM_MODEL -> gpt-4o-mini.
      # URL is diagnoser-specific ON PURPOSE: LLM_SERVICE_URL is also read by
      # internal/workers/intent.go with a DIFFERENT fallback (planner:5011), so
      # passing that name through here would silently repoint the intent worker.
      RSYNC_LLM_DIAGNOSER_MODEL: "${RSYNC_LLM_DIAGNOSER_MODEL:-}"
      RSYNC_LLM_DIAGNOSER_URL: "${RSYNC_LLM_DIAGNOSER_URL:-}"
      # SECURITY: ENCRYPTION_KEY must be 32+ characters. Generate with: openssl rand -base64 32
      # WARNING: Changing this key will make existing encrypted connection configs unreadable!
      # For production: Override via .env file or environment variable
      ENCRYPTION_KEY: "${ENCRYPTION_KEY:-dev-encryption-key-32-bytes-long!!}"
      TOOLS_DIR: "/app/shared/mcp-connectors"
      # Self-healing MCP deployment (tool-generator has Docker access via docker.sock)
      TOOL_GENERATOR_URL: "http://tool-generator:5010"
      # Sentinel core-service probes (sentinel/service_probes.go). Each is probed only
      # when its address is set; after 3 failed 30s checks the service is recorded
      # unhealthy and every admin gets an INFRASTRUCTURE_DOWN alert. Redis is always
      # probed, at the address the correlation client uses. TEMPORAL_ADDRESS is also the
      # (parked) native Temporal worker's dial address, whose default is this same value.
      TEMPORAL_ADDRESS: "temporal:7233"
      SENTINEL_PROBE_TEMPORAL_ADAPTER_URL: "http://temporal-adapter:8082"
      SENTINEL_PROBE_CONNECTOR_DEPLOYER_URL: "http://connector-deployer:5011"
      SENTINEL_PROBE_LLM_SERVICE_URL: "http://llm-service:5000"
      # Redis Configuration (State Store)
      REDIS_HOST: "redis"
      REDIS_PORT: "6379"
      REDIS_DB: "0"
      REDIS_PASSWORD: "${REDIS_PASSWORD:-}"
      # Correlation Store (V2 Request/Reply Pattern)
      # Must be >= longest expected request processing time to avoid duplicate claims.
      CORRELATION_CLAIM_TTL: "1h"
      # Serialization (Demo-2 default: JSON; Schema Registry is optional behind a profile)
      KAFKA_USE_AVRO: "false"
      SCHEMA_REGISTRY_URL: "http://schema-registry:8080"
      AVRO_AUTO_REGISTER_SCHEMAS: "false"
      # Logging Configuration (JSON for log collection)
      LOG_FORMAT: "json"
      LOG_LEVEL: "info"
      ENVIRONMENT: "development"
      # Scheduler is deprecated (schedules are managed via API Gateway + Temporal).
      # Disable to avoid legacy DB queries against removed columns.
      ENABLE_SCHEDULER: "false"
      # OpenTelemetry Configuration (exports to the bundled OTel Collector)
      OTEL_EXPORTER_OTLP_ENDPOINT: "otel-collector:4317"
      OTEL_SERVICE_NAME: "orchestrator"
      OTEL_ENABLED: "${OTEL_ENABLED:-false}"
      OTEL_TRACES_EXPORTER: "otlp"
      OTEL_METRICS_EXPORTER: "otlp"
      # Enable Debezium topic consumer that emits "captured" CDC table stats
      # (incoming inserts/updates/deletes), so UI can show incoming vs applied like DMS.
      ENABLE_CDC_TABLE_STATS: "true"
      # User MCP connectors run on the rsync-ai-mcp network (separate from internal services)
      DOCKER_NETWORK: "${STACK_PREFIX:-rsync-ai}-mcp"
      # Drives isolated-stack MCP container-name discovery in the orchestrator
      # (infra_preflight health URLs + mcp.containerNameCandidates). Default
      # rsync-ai; an isolated CI stack sets rsync-ci.
      STACK_PREFIX: "${STACK_PREFIX:-rsync-ai}"
      # Phase 3: Temporal-native executor dispatch — ⛔ PARKED, keep these OFF.
      # The native path is dead scaffolding: nothing populates the workflow's
      # ExecutorDispatch field, so all pipelines use the correlation path and the
      # native worker (if started) only idles. EXECUTOR_DISPATCH is not even read in
      # code. Leave both at their defaults below.
      EXECUTOR_NATIVE_WORKER: "${EXECUTOR_NATIVE_WORKER:-false}"
      EXECUTOR_DISPATCH: "${EXECUTOR_DISPATCH:-correlation}"
      # Export page size (rows/batch). Empty = code default (10000). Raise to cut
      # staging round-trips for large tables; opt-in per CAPABILITIES (batch-size lever).
      EXECUTOR_TABLE_BATCH_ROWS: "${EXECUTOR_TABLE_BATCH_ROWS:-}"
      # Batches exported per dispatch before signaling needs_continuation (chunk
      # boundary = this × EXECUTOR_TABLE_BATCH_ROWS rows). Empty = code default (200).
      # Lower it to force the chunked-continuation resume path on small tables.
      EXECUTOR_TABLE_CHUNK_BATCHES: "${EXECUTOR_TABLE_CHUNK_BATCHES:-}"
      # Hard per-table runaway ceiling (rows). Empty/0 = unlimited; the chunk loop
      # runs to natural EOF. Set >0 only to bound a suspected pagination bug.
      EXECUTOR_TABLE_MAX_ROWS: "${EXECUTOR_TABLE_MAX_ROWS:-}"
      # Which MCP connector containers exist on THIS host, for sentinel's connector
      # health sweep (see the docker-socket-proxy service above). The sweep used to
      # infer "not deployed here" from a name failing to resolve, which is only sound
      # on a resolver that answers an unknown name with a clean NXDOMAIN. On a GCP VM
      # it is not: /etc/resolv.conf carries `search ....internal` with ndots:0, Docker's
      # resolver forwards those variants upstream, and one non-NXDOMAIN answer left
      # every undeployed connector looking like a fault — 18 of them on prod. The
      # Docker API needs no resolver semantics, and `all=true` additionally tells a
      # stopped container (a real fault) from one that was never here, which DNS
      # cannot. Unset (quickstart/self-host runs no proxy) → the resolver fallback,
      # which now records nothing rather than guessing.
      MCP_DOCKER_API_URL: "http://docker-socket-proxy:2375"
    depends_on:
      postgres:
        condition: service_healthy
        # Other half of docker-compose.byo-postgres.yml — see that file's header:
        # profiles: without required:false = "depends on undefined service".
        required: false
      kafka:
        condition: service_healthy
        # Other half of docker-compose.byo-kafka.yml — see that file's header:
        # profiles: without required:false = "depends on undefined service".
        required: false
      redis:
        condition: service_healthy
      # Read-only Docker socket proxy for the connector-deployment census above.
      docker-socket-proxy:
        condition: service_started
    networks:
      - default
      - rsync-ai-mcp
    volumes:
      - ./shared/mcp-connectors:/app/shared/mcp-connectors
    extra_hosts:
      - "host.docker.internal:host-gateway"
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"

  # API Gateway (Go)
  api-gateway:
    build:
      context: .
      dockerfile: api-gateway/Dockerfile
      args: *build-info
    container_name: rsync-ai-api-gateway
    # main() exits when the database is unreachable at boot (db.Init has already
    # retried for DB_CONNECT_TIMEOUT). The restart is what turns that exit into a
    # recovery once Postgres is up; without it the container just stays down.
    restart: unless-stopped
    ports:
      - "${RSYNC_BIND_ADDR:-127.0.0.1}:${RSYNC_HP_API_GATEWAY:-5001}:8080" # Public API port (5001 to avoid macOS AirPlay on 5000)
    environment:
      # Kafka client security for a BYO/managed broker. See x-kafka-security at
      # the top of this file; every key defaults to empty and empty means unset,
      # so the bundled broker is unaffected.
      <<: *kafka-security
      DATABASE_URL: "postgres://user:password@postgres:5432/pipeline_db?sslmode=disable"
      KAFKA_BROKERS: "${KAFKA_BROKERS:-kafka:29092}"
      # Namespace for every Kafka topic this product creates, so an operator can
      # tell them apart from other applications' topics on a shared (BYO-Kafka)
      # cluster. Every service that touches Kafka must agree on this value: a
      # producer and a consumer resolving different names do not error, the
      # consumer just blocks on a topic nobody writes. `-` not `:-` on purpose --
      # an explicit empty value means "disable" (keep pre-prefix topic names) and
      # `:-` would silently turn that back into the default.
      KAFKA_TOPIC_PREFIX: "${KAFKA_TOPIC_PREFIX-rsync.}"
      # Dev/E2E: allow internal-only connectors (e.g. MinIO) to be used from UI/tests.
      RSYNC_ALLOW_INTERNAL_CONNECTORS: "true"
      # Live MCP container status via the read-only docker-socket-proxy (see the
      # docker-socket-proxy service above). Without this, api-gateway can't read
      # /var/run/docker.sock as non-root, so the catalog reports every connector
      # as "auto_deploys_on_use" and the UI shows "Running (0)". Unset → falls
      # back to dialing the local Unix socket directly.
      MCP_DOCKER_API_URL: "http://docker-socket-proxy:2375"
      # Admin panel access (beta). Comma-separated allowlist of emails.
      # Example: RSYNC_ADMIN_EMAILS=alice@company.com,bob@company.com
      RSYNC_ADMIN_EMAILS: "${RSYNC_ADMIN_EMAILS:-}"
      # CDC auto-pickup: a pipeline selected as "this whole database/schema"
      # keeps that meaning — the api-gateway re-expands the selection rule on
      # this interval and adds source tables created since. Default ON; set
      # false to freeze every CDC pipeline at its current table list. The
      # interval is floored at 60s (discovery runs once per CDC pipeline
      # per tick).
      CDC_TABLE_AUTOPICKUP_ENABLED: "${CDC_TABLE_AUTOPICKUP_ENABLED:-true}"
      CDC_TABLE_WATCH_INTERVAL_SECONDS: "${CDC_TABLE_WATCH_INTERVAL_SECONDS:-300}"
      # Temporal is required for the agentic pipeline stage-by-stage execution + progress events.
      # Without this, the gateway falls back to legacy direct-Kafka which doesn't drive the V2 UI timeline.
      TEMPORAL_ADDRESS: "temporal:7233"
      ORCHESTRATOR_URL: "http://orchestrator:8080"
      AGENT_ORCHESTRATOR_URL: "http://orchestrator:8080"
      TOOL_GENERATOR_URL: "http://tool-generator:5010"
      JWT_SECRET: "dev_secret_key_change_in_prod"
      # V2 is the ONLY workflow (V1 deleted)
      REDIS_ADDRESS: "redis:6379"
      REDIS_PASSWORD: "${REDIS_PASSWORD:-}"
      # SECURITY: Must match api-gateway ENCRYPTION_KEY for shared credential access
      ENCRYPTION_KEY: "${ENCRYPTION_KEY:-dev-encryption-key-32-bytes-long!!}"
      # CSRF protection (double-submit). Default off in dev; enable in staging with RSYNC_CSRF_ENFORCE=true.
      RSYNC_CSRF_ENFORCE: "${RSYNC_CSRF_ENFORCE:-false}"
      # Verifies X-Internal-Secret on internal S2S endpoints (e.g. the scheduled
      # model-run endpoint the temporal-adapter calls). Unset → those endpoints 503.
      INTERNAL_SERVICE_SECRET: "${INTERNAL_SERVICE_SECRET:-}"
      # Read-only mirror of the orchestrator's schema-drift switch, so the pipeline's
      # "Schema change alerts" card can say when detection is off for the whole
      # installation (schema_evolution.go schemaDriftDetectorEnabled). Must stay
      # byte-identical to the orchestrator line; a census test pins it.
      RSYNC_SCHEMA_DRIFT_ENABLED: "${RSYNC_SCHEMA_DRIFT_ENABLED:-false}"
      # Serialization (Demo-2 default: JSON; Schema Registry is optional behind a profile)
      KAFKA_USE_AVRO: "false"
      SCHEMA_REGISTRY_URL: "http://schema-registry:8080"
      AVRO_AUTO_REGISTER_SCHEMAS: "false"
      # Logging Configuration (JSON for log collection)
      LOG_FORMAT: "json"
      LOG_LEVEL: "info"
      ENVIRONMENT: "development"
      # OAuth Configuration
      OAUTH_CALLBACK_URL: "http://localhost:5001/oauth/callback"
      GITHUB_CLIENT_ID: "${GITHUB_CLIENT_ID}"
      GITHUB_CLIENT_SECRET: "${GITHUB_CLIENT_SECRET}"
      GOOGLE_CLIENT_ID: "${GOOGLE_CLIENT_ID}"
      GOOGLE_CLIENT_SECRET: "${GOOGLE_CLIENT_SECRET}"
      HUBSPOT_CLIENT_ID: "${HUBSPOT_CLIENT_ID}"
      HUBSPOT_CLIENT_SECRET: "${HUBSPOT_CLIENT_SECRET}"
      SALESFORCE_CLIENT_ID: "${SALESFORCE_CLIENT_ID}"
      SALESFORCE_CLIENT_SECRET: "${SALESFORCE_CLIENT_SECRET}"
      PIPEDRIVE_CLIENT_ID: "${PIPEDRIVE_CLIENT_ID}"
      PIPEDRIVE_CLIENT_SECRET: "${PIPEDRIVE_CLIENT_SECRET}"
      SLACK_CLIENT_ID: "${SLACK_CLIENT_ID}"
      SLACK_CLIENT_SECRET: "${SLACK_CLIENT_SECRET}"
      SHOPIFY_CLIENT_ID: "${SHOPIFY_CLIENT_ID:-}"
      SHOPIFY_CLIENT_SECRET: "${SHOPIFY_CLIENT_SECRET:-}"
      # Notifier (Pillar 6) — delivers pipeline failure alerts to Slack + email.
      # Leave empty in dev to keep notifications in "persist-only" mode
      # (queryable via UI, no external delivery).
      NOTIFIER_SLACK_WEBHOOK_URL: "${NOTIFIER_SLACK_WEBHOOK_URL:-}"
      # Slack one-click drift approvals (inbound interactivity). SIGNING_SECRET
      # verifies Slack request signatures; BOT_TOKEN (users:read.email) maps the
      # clicking Slack user to their rsync.ai account by email. Both unset ⇒ the
      # inbound receiver is inert and drift alerts keep the plain link button.
      SLACK_SIGNING_SECRET: "${SLACK_SIGNING_SECRET:-}"
      SLACK_BOT_TOKEN: "${SLACK_BOT_TOKEN:-}"
      SMTP_HOST: "${SMTP_HOST:-}"
      SMTP_PORT: "${SMTP_PORT:-587}"
      SMTP_USER: "${SMTP_USER:-}"
      SMTP_PASSWORD: "${SMTP_PASSWORD:-}"
      SMTP_FROM: "${SMTP_FROM:-}"
      # Email verification (Resend). Unset → signups auto-verify (dev default).
      # To TEST email verification locally: set RESEND_API_KEY + RESEND_FROM_ADDR
      # in the root .env (must be a Resend-verified domain), then recreate api-gateway.
      RESEND_API_KEY: "${RESEND_API_KEY:-}"
      RESEND_FROM_ADDR: "${RESEND_FROM_ADDR:-}"
      APP_BASE_URL: "${APP_BASE_URL:-http://localhost:3000}"
      # OpenTelemetry Configuration (exports to the bundled OTel Collector)
      OTEL_EXPORTER_OTLP_ENDPOINT: "otel-collector:4317"
      OTEL_SERVICE_NAME: "api-gateway"
      OTEL_ENABLED: "${OTEL_ENABLED:-false}"
      OTEL_TRACES_EXPORTER: "otlp"
      OTEL_METRICS_EXPORTER: "otlp"
      # Monitoring UI feature flags (V1)
      FEATURE_MONITORING_OVERVIEW: "true"
      FEATURE_MONITORING_INFRA: "true"
      FEATURE_MONITORING_TRACES: "true"
      # JIT Integration: Path to generated tools
      TOOLS_PATH: "/app/shared/mcp-connectors"
    depends_on:
      postgres:
        condition: service_healthy
        # Other half of docker-compose.byo-postgres.yml — see that file's header:
        # profiles: without required:false = "depends on undefined service".
        required: false
      kafka:
        condition: service_healthy
        # Other half of docker-compose.byo-kafka.yml — see that file's header:
        # profiles: without required:false = "depends on undefined service".
        required: false
      redis:
        condition: service_healthy
      # Wait for Temporal to accept connections so the startup client.Dial
      # succeeds instead of losing the cold-boot race (workflows disabled).
      # Belt-and-suspenders: the gateway also retries + lazily reconnects.
      temporal:
        condition: service_healthy
      orchestrator:
        condition: service_started
      # Read-only Docker socket proxy for live MCP container status.
      docker-socket-proxy:
        condition: service_started
    networks:
      - default
      - rsync-ai-mcp
    volumes:
      # JIT Integration: Read access to generated tools
      - ./shared/mcp-connectors:/app/shared/mcp-connectors:ro
      # NOTE: live docker container status (docker_status=running, port,
      # container_name) is NOT reported by the catalog endpoint because
      # api-gateway runs as a non-root user that can't read
      # /var/run/docker.sock. Mounting the socket here returns
      # "permission denied" without also setting `user: root` or doing
      # GID mapping — and we don't want to elevate api-gateway just for
      # an informational field. The catalog instead reports
      # docker_status="auto_deploys_on_use" for connectors whose
      # containers will be started on demand by the orchestrator. The
      # full live-status feature is tracked as a follow-up that would
      # route the docker query through tool-generator (which already
      # has its own RW socket mount with the appropriate privileges).
    extra_hosts:
      - "host.docker.internal:host-gateway"
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"

  # LLM Gateway (Python) - Internal service, no host port needed
  llm-service:
    build:
      context: ./llm-service
      dockerfile: Dockerfile
      args: *build-info
    container_name: rsync-ai-llm-service
    env_file:
      # Gitignored local override. `required: false` so `docker compose config`
      # (e.g. the e2e/staging wiring guards) doesn't hard-fail on a runner that
      # has no local .env; the file is still loaded wherever it exists.
      - path: ./llm-service/.env
        required: false
    environment:
      # Kafka client credentials. llm-service runs the PII-scanner consumer and
      # its producer (src/agents/pii_scanner/kafka_consumer.py via
      # src/utils/kafka_security.py), started as a background task at
      # src/gateway/main.py:770 whose failure is swallowed into a warning — so
      # an unauthenticated or unreachable broker here produces NO error surface
      # at all, just a scanner that never runs. Without the anchor a BYO broker
      # is dialled anonymously over PLAINTEXT.
      <<: *kafka-security
      # The address belongs here for a second reason: docker-compose.byo-kafka.yml
      # REMOVES the bundled `kafka` service, so an operator who sets
      # KAFKA_BROKERS=my-msk:9098 redirected every other service while this one
      # kept the code default `kafka:29092` — a host that no longer exists.
      KAFKA_BROKERS: ${KAFKA_BROKERS:-kafka:29092}
      # Namespace for every Kafka topic this product creates. A producer and a
      # consumer resolving different names do not error, the consumer just blocks
      # on a topic nobody writes — so this must agree with the five services that
      # already declare it. `-` not `:-` on purpose: an explicit empty value means
      # "disable" (keep pre-prefix topic names) and `:-` would silently turn that
      # back into the default.
      KAFKA_TOPIC_PREFIX: ${KAFKA_TOPIC_PREFIX-rsync.}
      PORT: "5000"
      LOG_FORMAT: json
      LOG_LEVEL: info
      ENVIRONMENT: development
      # Sent as X-Internal-Secret on service-to-service calls to the orchestrator
      # topology API (requirePrincipal). Empty = topology calls 401 in production.
      INTERNAL_SERVICE_SECRET: ${INTERNAL_SERVICE_SECRET:-}
      # Control-plane DB for best-effort LLM cost/usage logging (llm_usage_events +
      # check_and_charge_user, migration 057). Without this the cost writer falls
      # back to an empty-password DSN and fails every write with "fe_sendauth: no
      # password supplied". Mirrors the api-gateway/orchestrator default; staging &
      # prod override DATABASE_URL via .env to point at Azure PostgreSQL.
      DATABASE_URL: ${DATABASE_URL:-postgres://user:password@postgres:5432/pipeline_db?sslmode=disable}
      # OpenTelemetry Configuration (exports to the bundled OTel Collector)
      OTEL_EXPORTER_OTLP_ENDPOINT: otel-collector:4317
      OTEL_SERVICE_NAME: llm-service
      OTEL_ENABLED: "${OTEL_ENABLED:-false}"
      OTEL_TRACES_EXPORTER: otlp
      OTEL_METRICS_EXPORTER: otlp
    # No host port exposed - accessed internally by planner/tool-generator via Docker network
    command: ["python", "-m", "src.gateway.main"]
    extra_hosts:
      - "host.docker.internal:host-gateway"
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"

  # Connector Deployer (SEC-H-02): the small, trusted, no-LLM microservice that
  # holds /var/run/docker.sock. tool-generator (which runs LLM/user-influenced
  # connector code) now DROPS root + the socket and calls this service over an
  # authenticated internal RPC (DEPLOYER_URL) to build/run JIT connector
  # containers. The security property is allow-by-construction: the deployer
  # builds the Docker `create` payload itself with a hardcoded-safe HostConfig,
  # so untrusted input is DATA — never a Docker API call — and no
  # Privileged/bind/cap field can be smuggled through. Because the HOST daemon
  # creates the container, it joins rsync-ai-mcp by name/alias and kafka-mcp-sink
  # still resolves JIT destination connectors there (flat-name reachability, the
  # decisive constraint). See docs/internal/jit-connector-sandboxing.md and
  # connector-deployer/CONTRACT.md.
  connector-deployer:
    build:
      context: ./connector-deployer
      args:
        # SEC-H-02 increment 2: lock the deployer's buildctl (client) to the buildkitd
        # sidecar's image tag in docker-compose.prod.yml. Bump both via one env knob.
        BUILDKIT_VERSION: ${BUILDKIT_VERSION:-v0.31.2}
    container_name: rsync-ai-connector-deployer
    environment:
      - ENVIRONMENT=development
      - LOG_FORMAT=json
      - LOG_LEVEL=info
      # Rotation for the connector containers THIS service creates -- compose's
      # `logging:` blocks only reach the services compose itself starts.
      - RSYNC_LOG_MAX_SIZE=${RSYNC_LOG_MAX_SIZE:-10m}
      - RSYNC_LOG_MAX_FILE=${RSYNC_LOG_MAX_FILE:-3}
      # S2S shared secret — callers send it as X-Internal-Secret. Unset in dev =>
      # the deployer allows with a warning (CONTRACT auth table); the prod overlay
      # fail-closes it with :? (mirrors the MinIO C3 pattern).
      - INTERNAL_SERVICE_SECRET=${INTERNAL_SERVICE_SECRET:-}
      # Pinned connector network used as NetworkMode — the SEC-M-06 isolation
      # boundary. Same value tool-generator passed as DOCKER_NETWORK.
      - DEPLOYER_DOCKER_NETWORK=${STACK_PREFIX:-rsync-ai}-mcp
      # "Also-join" network for kafka-mcp-sink reachability; == the primary net
      # here, so the also-join is a no-op (matches tool-generator's prior
      # MCP_SHARED_NETWORK default of rsync-ai-mcp).
      - MCP_SHARED_NETWORK=${STACK_PREFIX:-rsync-ai}-mcp
      # Connector container-name prefix (<STACK_PREFIX>-<id>-vX-Y-Z-mcp). Without it
      # an isolated stack's compose-managed connectors are not recognised as
      # protected, so a stopped one can never be restarted.
      - STACK_PREFIX=${STACK_PREFIX:-rsync-ai}
      # Named volume for the OAuth TokenManager, mounted at /root/.rsync-ai below.
      # SAME value tool-generator passed to connectors (docker_builder.py default),
      # so persisted connector OAuth tokens survive the tool-generator → deployer move.
      - OAUTH_TOKENS_VOLUME_NAME=rsync-ai-oauth-tokens
      # Passphrase the TokenManager derives its AES-GCM key from (PBKDF2, random
      # per-record salt). Without it every token write raised and was swallowed,
      # so the volume above stayed empty while store_token() logged success —
      # OAUTH_ENCRYPTION_KEY, which takes precedence, is set by no compose file
      # and no chart. Reusing ENCRYPTION_KEY keeps this to one secret per
      # deployment instead of a second one nothing remembers to set.
      - ENCRYPTION_KEY=${ENCRYPTION_KEY:-dev-encryption-key-32-bytes-long!!}
    # THE trusted socket holder: runs as root (0:0) and mounts the RW docker.sock.
    # This is the one small, audited surface that speaks the Docker API — moved
    # OFF tool-generator, which runs untrusted connector code.
    user: "0:0"
    networks:
      - default
      - rsync-ai-mcp
    volumes:
      # RW docker socket — build images + create/start/stop/rm JIT connector containers.
      - /var/run/docker.sock:/var/run/docker.sock
      # Connector artifacts, READ-ONLY: the deployer only reads build contexts,
      # never writes them.
      - ./shared/mcp-connectors:/app/shared/mcp-connectors:ro
      # docker-compose.mcp.yml, READ-ONLY: protected-compose-container detection
      # (never rebuild/tear down a compose-managed connector serving live CDC).
      - ./docker-compose.mcp.yml:/app/docker-compose.mcp.yml:ro
      # The SAME OAuth TokenManager volume tool-generator used, for token persistence.
      - tool_generator_oauth_tokens:/root/.rsync-ai
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"

  # Connector-tree ownership init (one-shot). SEC-H-02 (#643) correctly dropped
  # tool-generator to the image's unprivileged appuser (uid 1000) — but that same
  # service still WRITES generated connector source into the RW-mounted
  # ./shared/mcp-connectors tree (public/<slug>/versions/… and oauth/providers.json).
  # On a root-owned checkout (prod lives at /root/rsync-ai) that tree is root:… 0755,
  # so uid 1000 can't create public/<slug> and connector GENERATION fails with
  # PermissionError (regression found on prod 2026-07-20; the #643 staging E2E only
  # exercised DEPLOYING existing connectors, never generating a new one). This root
  # one-shot chowns the tree to uid 1000 before tool-generator starts, so a fresh
  # checkout / re-provision can't silently re-break generation.
  #
  # NOT a SEC-H-02 regression: this container holds NO docker.sock and touches ONLY
  # the connector tree. The host-root surface #643 removed is the docker socket,
  # which lives solely on connector-deployer — this init never goes near it.
  # (OSS/quickstart are unaffected: they mount the tree :ro from a named volume
  # seeded by connector-seed, so they never take this base service.)
  connector-fs-init:
    image: busybox:1.37.0
    container_name: rsync-ai-connector-fs-init
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"
    user: "0:0"
    command: ["chown", "-R", "1000:1000", "/app/shared/mcp-connectors"]
    volumes:
      - ./shared/mcp-connectors:/app/shared/mcp-connectors
    restart: "no"

  # Tool Generator Agent
  tool-generator:
    build:
      context: ./llm-service
      dockerfile: Dockerfile
    container_name: rsync-ai-tool-generator
    env_file:
      # Gitignored local override. `required: false` so `docker compose config`
      # (e.g. the e2e/staging wiring guards) doesn't hard-fail on a runner that
      # has no local .env; the file is still loaded wherever it exists.
      - path: ./llm-service/.env
        required: false
    environment:
      - PORT=5010
      - TOOLS_DIR=/app/shared/mcp-connectors
      # Default: enabled for local dev. In managed mode tool-generator asks the
      # connector-deployer (DEPLOYER_URL) to build images and start containers on
      # the rsync-ai-mcp network in response to /v1/deploy (called by the
      # orchestrator's Preflight agent when a required MCP container isn't already
      # running). tool-generator no longer touches docker.sock itself (SEC-H-02):
      # the connector-deployer is the sole trusted socket holder. Override to
      # "false" for environments where MCPs are managed by an external
      # docker-compose.mcp.yml stack (docker-compose.prod.yml ships that pattern).
      - RSYNC_MANAGED_CONNECTORS=${RSYNC_MANAGED_CONNECTORS:-true}
      # SEC-H-02: JIT build/run is delegated to the connector-deployer over an
      # authenticated internal RPC. INTERNAL_SERVICE_SECRET is sent as
      # X-Internal-Secret; unset in dev => no-op (deployer allows with a warning).
      - DEPLOYER_URL=http://connector-deployer:5011
      - INTERNAL_SERVICE_SECRET=${INTERNAL_SERVICE_SECRET:-}
      - LOG_FORMAT=json
      - LOG_LEVEL=info
      - ENVIRONMENT=development
      # LLM Service URL (internal Docker network)
      - LLM_GATEWAY_URL=http://llm-service:5000
      # Context7 MCP Service URL (internal Docker network)
      - CONTEXT7_MCP_URL=http://context7-mcp:8080
      # Guardrail: require a strong match between connector name and resolved Context7 docs
      # to avoid false positives for random strings (tunable in prod).
      - CONTEXT7_VALIDATE_MIN_SCORE=0.90
      - CONTEXT7_RESOLVE_MIN_SCORE=0.85
      # LLM Provider Configuration
      # OPENAI_API_KEY must be provided via env_file (./llm-service/.env) or your shell environment.
      - OLLAMA_BASE_URL=http://host.docker.internal:11434/v1
      - OLLAMA_MODEL=${OLLAMA_MODEL:-qwen2.5:7b}
      - CONTEXT7_MCP_URL=http://context7-mcp:8080
      # JIT-deployed (user/LLM-generated) connectors run on the DEDICATED rsync-ai-mcp
      # network — isolated from the internal services that sit on rsync-ai_default
      # (kafka, temporal, redis, postgres) so untrusted connector code cannot reach
      # them by L3 (SEC-M-06). orchestrator, api-gateway, tool-generator, AND
      # kafka-mcp-sink all bridge into rsync-ai-mcp (see each service's networks:
      # block), so they still resolve JIT + compose-MCP connectors by container name
      # / alias there. This matches the compose-MCP topology (the generator script
      # scripts/mcp_generate_compose.py joins rsync-ai-mcp) and the OSS/quickstart
      # stacks (which already set DOCKER_NETWORK=rsync-ai-mcp). An earlier revision
      # put JIT containers on rsync-ai_default to fix a sink "no such host" bug; that
      # workaround is obsolete now that kafka-mcp-sink bridges into rsync-ai-mcp
      # (see its networks: block above).
      - DOCKER_NETWORK=${STACK_PREFIX:-rsync-ai}-mcp
      # Connector container-name prefix: the orchestrator looks for
      # <STACK_PREFIX>-<id>-vX-Y-Z-mcp, so /v1/deploy must start that name.
      - STACK_PREFIX=${STACK_PREFIX:-rsync-ai}
      # OpenTelemetry Configuration (off by default; set OTEL_ENABLED=true to export to your own collector)
      - OTEL_EXPORTER_OTLP_ENDPOINT=otel-collector:4317
      - OTEL_SERVICE_NAME=tool-generator
      - OTEL_ENABLED=${OTEL_ENABLED:-false}
      - OTEL_TRACES_EXPORTER=otlp
      - OTEL_METRICS_EXPORTER=otlp
    # No host port — tool-generator is internal only, accessed via api-gateway
    # Entrypoint is src.lifecycle.main, NOT src.agents.tool_generator.service: the
    # latter is the connector-GENERATION service and is not part of this repo, so
    # its module would be missing and the container would die on import. This one
    # serves the connector-lifecycle surface the data plane calls (/v1/deploy).
    command: ["python", "-m", "src.lifecycle.main"]
    # SEC-H-02: tool-generator runs the image's default unprivileged appuser
    # (uid 1000) — it no longer needs root, because it no longer holds the
    # docker socket. All JIT build/run goes through the connector-deployer
    # (DEPLOYER_URL), the sole trusted socket holder. This is the security win:
    # the one service that runs LLM/user-influenced connector code is no longer
    # host-root-equivalent.
    networks:
      - default
      - rsync-ai-mcp
    volumes:
      - ./shared/mcp-connectors:/app/shared/mcp-connectors
      - ./docker-compose.mcp.yml:/app/docker-compose.mcp.yml
      # Persist OAuth TokenManager storage across container restarts. The
      # connector-deployer mounts this SAME named volume (SEC-H-02), so OAuth
      # tokens stay readable by the connectors it spawns.
      # NOTE (SEC-H-02): no docker.sock mount here anymore — managed-mode JIT
      # build/run is delegated to the connector-deployer over its internal RPC.
      - tool_generator_oauth_tokens:/root/.rsync-ai
    # Map form (not the short list) so connector-fs-init can gate on completion:
    # tool-generator must not start until the connector tree is chowned to uid 1000,
    # or its first generation write races the init and fails with PermissionError.
    depends_on:
      llm-service:
        condition: service_started
      context7-mcp:
        condition: service_started
      connector-deployer:
        condition: service_started
      connector-fs-init:
        condition: service_completed_successfully
    extra_hosts:
      - "host.docker.internal:host-gateway"
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"


  # Planner Agent
  planner:
    build:
      context: ./llm-service
      dockerfile: Dockerfile
    container_name: rsync-ai-planner
    env_file:
      # Gitignored local override. `required: false` so `docker compose config`
      # (e.g. the e2e/staging wiring guards) doesn't hard-fail on a runner that
      # has no local .env; the file is still loaded wherever it exists.
      - path: ./llm-service/.env
        required: false
    environment:
      # Kafka client security for a BYO/managed broker. See x-kafka-security at
      # the top of this file; every key defaults to empty and empty means unset,
      # so the bundled broker is unaffected.
      <<: *kafka-security
      PORT: "5011"
      LOG_FORMAT: "json"
      LOG_LEVEL: "info"
      ENVIRONMENT: "development"
      TOOL_GENERATOR_URL: "http://tool-generator:5010"
      API_GATEWAY_URL: "http://api-gateway:8080"
      LLM_GATEWAY_URL: "http://llm-service:5000"
      KAFKA_BOOTSTRAP_SERVERS: "${KAFKA_BROKERS:-kafka:29092}"
      # Namespace for every Kafka topic this product creates, so an operator can
      # tell them apart from other applications' topics on a shared (BYO-Kafka)
      # cluster. Every service that touches Kafka must agree on this value: a
      # producer and a consumer resolving different names do not error, the
      # consumer just blocks on a topic nobody writes. `-` not `:-` on purpose --
      # an explicit empty value means "disable" (keep pre-prefix topic names) and
      # `:-` would silently turn that back into the default.
      KAFKA_TOPIC_PREFIX: "${KAFKA_TOPIC_PREFIX-rsync.}"
      # JIT Integration: Planner needs access to check connector existence
      CONNECTORS_DIR: "/app/shared/mcp-connectors"
      # OpenTelemetry Configuration (off by default; set OTEL_ENABLED=true to export to your own collector)
      OTEL_EXPORTER_OTLP_ENDPOINT: "otel-collector:4317"
      OTEL_SERVICE_NAME: "planner"
      OTEL_ENABLED: "${OTEL_ENABLED:-false}"
      OTEL_TRACES_EXPORTER: "otlp"
      OTEL_METRICS_EXPORTER: "otlp"
      # Sent to the orchestrator's /api/v1/topology (strategies.py), which
      # requirePrincipal gates; without it plan-time topic creation gets 401.
      INTERNAL_SERVICE_SECRET: "${INTERNAL_SERVICE_SECRET:-}"
    volumes:
      # JIT Integration: Shared volume for connector existence checks
      - ./shared/mcp-connectors:/app/shared/mcp-connectors:ro
    ports:
      - "127.0.0.1:${RSYNC_HP_PLANNER:-8001}:5011"
    command: ["python", "-m", "src.agents.planner.service"]
    # Long-form depends_on: the short list form cannot carry `required: false`,
    # and the `kafka` entry below needs it — see docker-compose.byo-kafka.yml.
    # `service_started` on every entry is exactly what the short form meant.
    depends_on:
      llm-service:
        condition: service_started
      tool-generator:
        condition: service_started
      api-gateway:
        condition: service_started
      kafka:
        condition: service_started
        # Other half of docker-compose.byo-kafka.yml — see that file's header:
        # profiles: without required:false = "depends on undefined service".
        required: false
    extra_hosts:
      - "host.docker.internal:host-gateway"
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"

  # Sentinel Agent (Infrastructure Watchdog)
  # Frontend (Next.js)
  frontend:
    container_name: rsync-ai-frontend
    build:
      context: ./frontend
      dockerfile: Dockerfile
    env_file:
      # Gitignored local override — optional so `docker compose config` works
      # on a runner without a local .env (loaded normally where it exists).
      - path: ./frontend/.env
        required: false
    ports:
      - "${RSYNC_BIND_ADDR:-127.0.0.1}:${RSYNC_HP_FRONTEND:-3000}:3000"
    environment:
      # Internal URLs for server-side fetches (within Docker network)
      - API_GATEWAY_INTERNAL_URL=http://api-gateway:8080
      - ORCHESTRATOR_INTERNAL_URL=http://orchestrator:8080
      # Server-only (NOT NEXT_PUBLIC): the RSC dashboard authenticates its
      # server-side call to the orchestrator with it (instrumentation.ts warns if unset).
      - INTERNAL_SERVICE_SECRET=${INTERNAL_SERVICE_SECRET:-}
      # Public URLs for client-side fetches (from browser via host)
      - NEXT_PUBLIC_API_URL=http://localhost:5001
      - NEXT_PUBLIC_BACKEND_ORCHESTRATOR_URL=http://localhost:8080
      - NEXT_PUBLIC_WS_URL=ws://localhost:5001/ws
      # Planner (agent) service exposed on host port 8001
      - NEXT_PUBLIC_LLM_SERVICE_URL=http://localhost:8001
      - NEXTAUTH_URL=http://localhost:3000
      # OpenTelemetry Configuration (for browser tracing)
      - NEXT_PUBLIC_OTEL_ENDPOINT=http://localhost:4318
      - NEXT_PUBLIC_OTEL_SERVICE_NAME=frontend
      # Monitoring UI feature flags (V1)
      - NEXT_PUBLIC_FEATURE_MONITORING_OVERVIEW=true
      - NEXT_PUBLIC_FEATURE_MONITORING_INFRA=true
      - NEXT_PUBLIC_FEATURE_MONITORING_TRACES=true
    depends_on:
      - api-gateway
    extra_hosts:
      - "host.docker.internal:host-gateway"
    logging:
      driver: json-file
      options:
        max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
        max-file: "${RSYNC_LOG_MAX_FILE:-3}"

volumes:
  postgres_data:
  kafka_data:
  redis_data:
  minio_data:
  blob_staging_data:
  # Debezium source secrets: written by debezium-mcp, read-only-mounted by
  # kafka-connect, so credentials resolve via FileConfigProvider instead of being
  # stored plaintext in the Kafka config topic.
  connect_secrets:
  tool_generator_oauth_tokens:
    name: rsync-ai-oauth-tokens

networks:
  # rsync-ai-mcp: dedicated network for USER MCP connector containers — both the
  # compose-generated connectors (docker-compose.mcp.yml) and the JIT-deployed ones
  # (tool-generator docker_builder.py). orchestrator, tool-generator, api-gateway,
  # AND kafka-mcp-sink bridge into it so they can reach those connectors by container
  # name / alias. Internal services (postgres, redis, kafka, temporal) do NOT join
  # this network, so user connectors on it cannot reach them by L3 — this is the
  # SEC-M-06 isolation boundary (JIT connectors are pinned here via DOCKER_NETWORK).
  rsync-ai-mcp:
    name: rsync-ai-mcp