# docker-compose.prod.yml # Production environment Docker Compose configuration # Requires a .env.prod file in the same directory. # Version note: Using '3.8' or higher is recommended for newer features like healthcheck conditions. # version: '3.8' services: # Nginx Reverse Proxy with Certbot nginx: image: nginx:1.25-alpine container_name: kiwiq_prod_nginx restart: unless-stopped env_file: - .env # environment: # - DOMAIN=${DOMAIN:-api.prod.kiwiq.ai} # - APP_HOST=${APP_HOST:-app} # - APP_PORT=${APP_PORT:-8000} ports: - "80:80" - "443:443" volumes: - ./docker/nginx/data/nginx:/etc/nginx/conf.d - ./docker/nginx/data/certbot/conf:/etc/letsencrypt - ./docker/nginx/data/certbot/www:/var/www/certbot command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g \"daemon off;\"'" networks: - prod_network depends_on: app: condition: service_started prefect-server: condition: service_started # Resource limits deploy: resources: limits: memory: 512M cpus: '0.5' reservations: memory: 256M cpus: '0.25' # Logging logging: driver: "json-file" options: max-size: "100m" max-file: "3" # Certbot for Let's Encrypt SSL certificates certbot: image: certbot/certbot container_name: kiwiq_prod_certbot restart: unless-stopped volumes: - ./docker/nginx/data/certbot/conf:/etc/letsencrypt - ./docker/nginx/data/certbot/www:/var/www/certbot entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'" networks: - prod_network # Resource limits (minimal for certbot) deploy: resources: limits: memory: 256M cpus: '0.25' reservations: memory: 128M cpus: '0.1' # Logging logging: driver: "json-file" options: max-size: "50m" max-file: "2" # Weaviate Vector Database Service weaviate: image: semitechnologies/weaviate:1.31.3 # Use specific version for production container_name: kiwiq_prod_weaviate restart: unless-stopped env_file: - .env environment: # Production Weaviate configuration QUERY_DEFAULTS_LIMIT: ${WEAVIATE_QUERY_DEFAULTS_LIMIT:-25} AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: ${WEAVIATE_ANONYMOUS_ACCESS_ENABLED:-true} # Disable for prod # AUTHENTICATION_APIKEY_ENABLED: ${WEAVIATE_APIKEY_ENABLED:-true} # Enable API key auth for prod # AUTHENTICATION_APIKEY_ALLOWED_KEYS: ${WEAVIATE_API_KEYS} # Set API keys from env PERSISTENCE_DATA_PATH: '/var/lib/weaviate' ENABLE_MODULES: ${WEAVIATE_ENABLE_MODULES:-text2vec-openai,generative-openai} DEFAULT_VECTORIZER_MODULE: ${WEAVIATE_DEFAULT_VECTORIZER_MODULE:-text2vec-openai} ASYNC_INDEXING: ${WEAVIATE_ASYNC_INDEXING:-true} OPENAI_APIKEY: ${OPENAI_API_KEY} # Required for OpenAI modules LOG_LEVEL: ${WEAVIATE_LOG_LEVEL:-info} # Production performance settings LIMIT_RESOURCES: ${WEAVIATE_LIMIT_RESOURCES:-true} GOMEMLIMIT: ${WEAVIATE_MEMORY_LIMIT:-4400MiB} # 5.7GB limit, ~90% of 6.25GB CLUSTER_HOSTNAME: "node1" RAFT_BOOTSTRAP_EXPECT: "1" RAFT_ENABLE_ONE_NODE_RECOVERY: "true" RAFT_BOOTSTRAP_TIMEOUT: "300" volumes: - weaviate_data:/var/lib/weaviate networks: - prod_network # No ports exposed externally - accessed via internal network only healthcheck: test: ["CMD", "wget", "--no-verbose", "--tries=3", "--spider", "http://localhost:8080/v1/.well-known/ready"] interval: 30s timeout: 10s retries: 3 start_period: 30s # Optional: Add production logging # logging: # driver: "json-file" # options: # max-size: "100m" # max-file: "3" # Resource limits (Adjusted for 4x workflow concurrency) deploy: resources: limits: memory: 4.75G # ⬆️ from 6G (+256MB) cpus: '2.5' # ⬆️ from 2.0 (25% increase for higher concurrency) reservations: memory: 3.75G # ⬆️ from 4G cpus: '1.5' # unchanged # Logging logging: driver: "json-file" options: max-size: "100m" max-file: "3" # Main Application Service (FastAPI/Uvicorn) app: build: context: . # Use the same Dockerfile as dev for now dockerfile: docker/Dockerfile.prod.app # Consider a separate Dockerfile.prod later for optimizations (e.g., no reload) # and to avoid running setup.sh on every start (see entrypoint note below). container_name: kiwiq_prod_app restart: unless-stopped # No ports exposed directly; Nginx handles external access. env_file: # Load production environment variables (includes full DB/Service URLs) - .env environment: # Override any dev-specific settings if necessary APP_ENV: PROD # Service URLs - ensure these match .env configuration # WEAVIATE_URL: "http://weaviate:8080" # Individual host vars are removed; rely on full URLs from .env.prod loaded above # POSTGRES_HOST: postgres # MONGO_HOST: mongo # RABBITMQ_HOST: rabbitmq # REDIS_HOST: redis # WEAVIATE_HOST: weaviate # RabbitMQ connection URL for event consumer PREFECT_API_AUTH_STRING: ${PREFECT_AUTH_STRING} RABBITMQ_URL: "amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672/" # Add other production-specific env vars if needed directly here volumes: # In production, typically you don't mount the source code directly. # The code should be copied into the image during the build. # Remove this line if using a dedicated Dockerfile.prod that copies the code. - ./:/app depends_on: # Explicitly depend on backend services weaviate: # condition: service_healthy condition: service_started # mongo dependency removed - using managed MongoDB cluster # postgres dependency removed - using managed PostgreSQL database rabbitmq: condition: service_started redis: condition: service_started # prefect-server: # condition: service_started # prefect-agent: # condition: service_started # Resource limits (Reduced for balanced allocation) deploy: resources: limits: memory: 2.5G cpus: '2' reservations: memory: 2G cpus: '1.0' # Logging logging: driver: "json-file" options: max-size: "200m" max-file: "5" networks: - prod_network # command: ["uvicorn", "services.kiwi_app.main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "4"] # NOTE on Entrypoint: Because this uses Dockerfile.prod.app, the ENTRYPOINT ["/app/setup.sh"] # from that file will run every time this container starts. This includes DB creation checks # and Alembic migrations. Ensure setup.sh is idempotent and consider if this is the desired # behavior for production restarts vs. running migrations as a separate deployment step. # Optional: Add production logging # logging: # driver: "awslogs" # ... # # MongoDB Database Service # mongo: # # Use a specific version tag in production (e.g., mongo:6.0) # image: mongo:latest # container_name: kiwiq_prod_mongo # restart: unless-stopped # # Needed for MONGO_ROOT_USERNAME, MONGO_ROOT_PASSWORD # env_file: # - .env # environment: # # Load credentials from .env.prod # MONGO_INITDB_ROOT_USERNAME: ${MONGO_ROOT_USERNAME} # MONGO_INITDB_ROOT_PASSWORD: ${MONGO_ROOT_PASSWORD} # # Persist data # volumes: # - mongo_data:/data/db # - mongo_config:/data/configdb # networks: # - prod_network # command: | # mongod # --wiredTigerCacheSizeGB=5 # --wiredTigerJournalCompressor=snappy # --wiredTigerCollectionBlockCompressor=snappy # --journalCommitInterval=50 # --maxConns=800 # --quiet # # ⬆️ from 2.5GB (optimized for 8GB memory) # # ⬆️ from 200 (4x for 60 concurrent workflows) # # No ports exposed externally # # Optional: Add basic healthcheck for Mongo if needed # # healthcheck: # # test: echo 'db.runCommand("ping").ok' | mongosh --quiet # # interval: 10s # # timeout: 10s # # retries: 5 # # Optional: Add production logging # # Resource limits (Upgraded for 4x workflow concurrency) # deploy: # resources: # limits: # memory: 8G # ⬆️ from 4G (doubled for higher concurrency) # cpus: '2.5' # ⬆️ from 2 (25% increase) # reservations: # memory: 5G # ⬆️ from 3.5G # cpus: '1.5' # # Logging # logging: # driver: "json-file" # options: # max-size: "100m" # max-file: "3" # # MongoDB Backup Service (Scheduled) # mongo-backup: # image: mongo:7.0 # container_name: kiwiq_prod_mongo_backup # restart: unless-stopped # env_file: # - .env # environment: # BACKUP_SCHEDULE: ${MONGO_BACKUP_SCHEDULE:-"0 2 * * *"} # Daily at 2 AM # BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} # volumes: # - ./backups/mongo:/backup # - ./scripts/mongo-backup.sh:/mongo-backup.sh:ro # entrypoint: ["/bin/sh", "-c"] # command: | # "apk add --no-cache dcron && \ # echo '${BACKUP_SCHEDULE} /mongo-backup.sh' | crontab - && \ # crond -f -l 2" # depends_on: # mongo: # condition: service_healthy # networks: # - prod_network # # Minimal resources for backup cron # deploy: # resources: # limits: # memory: 512M # cpus: '0.5' # reservations: # memory: 256M # cpus: '0.25' # # Logging # logging: # driver: "json-file" # options: # max-size: "50m" # max-file: "2" # # PostgreSQL Database Service # postgres: # # Use a specific version tag in production (e.g., postgres:15) # image: postgres:latest # container_name: kiwiq_prod_postgres # restart: unless-stopped # # Needed for POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_DB # env_file: # - .env # environment: # POSTGRES_USER: ${POSTGRES_USER} # POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} # # Main DB, others created by setup.sh if needed # POSTGRES_DB: ${POSTGRES_DB} # # POSTGRES_DATABASES is needed if setup.sh in the app container creates multiple DBs. # # Ensure it's defined in .env.prod # # POSTGRES_DATABASES: ${POSTGRES_DATABASES} # # Optimized production configuration (4x workflow concurrency) # command: [ # "postgres", # # Connection settings (increased for 60 concurrent workflows) # "-c", "max_connections=600", # ⬆️ from 300 (4x workflows need more connections) # "-c", "shared_buffers=1GB", # ⬆️ from 640MB # "-c", "effective_cache_size=3GB", # ⬆️ from 1.5GB # "-c", "work_mem=4MB", # "-c", "maintenance_work_mem=512MB", # # Checkpoint settings # "-c", "checkpoint_completion_target=0.9", # "-c", "wal_buffers=16MB", # "-c", "default_statistics_target=100", # # Write performance # "-c", "synchronous_commit=on", # "-c", "wal_level=replica", # "-c", "max_wal_size=4GB", # "-c", "min_wal_size=1GB", # # Query performance (increased for higher concurrency) # "-c", "random_page_cost=1.1", # "-c", "effective_io_concurrency=400", # ⬆️ from 200 (doubled for higher concurrent I/O) # # Logging # "-c", "log_min_duration_statement=5000", # "-c", "log_connections=on", # "-c", "log_disconnections=on", # "-c", "log_statement=mod", # "-c", "log_line_prefix='%t [%p]: [%l-1] db=%d,user=%u '", # ] # # Resource limits # deploy: # resources: # limits: # memory: 4G # ⬆️ from 2.5G (+500MB) # cpus: '2.0' # unchanged # reservations: # memory: 3G # ⬆️ from 1.5G # cpus: '0.75' # unchanged # # Production logging # logging: # driver: "json-file" # options: # max-size: "100m" # max-file: "3" # # Persist data # volumes: # - postgres_data:/var/lib/postgresql/data # networks: # - prod_network # # No ports exposed externally # healthcheck: # # Test if Postgres is ready to accept connections # test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB} -h localhost"] # interval: 10s # timeout: 5s # retries: 5 # # Give some time for initialization # start_period: 10s # # Optional: Add production logging # # PostgreSQL Backup Service # postgres-backup: # image: postgres:15 # container_name: kiwiq_prod_postgres_backup # restart: unless-stopped # env_file: # - .env # environment: # PGPASSWORD: ${POSTGRES_PASSWORD} # BACKUP_SCHEDULE: ${POSTGRES_BACKUP_SCHEDULE:-"0 3 * * *"} # Daily at 3 AM # BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} # volumes: # - ./backups/postgres:/backup # - ./scripts/postgres-backup.sh:/postgres-backup.sh:ro # entrypoint: ["/bin/sh", "-c"] # command: | # "apt-get update && apt-get install -y cron && \ # echo '${BACKUP_SCHEDULE} /postgres-backup.sh' | crontab - && \ # cron -f" # depends_on: # postgres: # condition: service_healthy # networks: # - prod_network # # Minimal resources # deploy: # resources: # limits: # memory: 512M # cpus: '0.5' # reservations: # memory: 256M # cpus: '0.25' # logging: # driver: "json-file" # options: # max-size: "50m" # max-file: "2" # RabbitMQ Messaging Service rabbitmq: # Use specific version (e.g., rabbitmq:3.11-management-alpine) image: rabbitmq:3-management-alpine container_name: kiwiq_prod_rabbitmq restart: unless-stopped # Needed for RABBITMQ_DEFAULT_USER, RABBITMQ_DEFAULT_PASS env_file: - .env environment: # CRITICAL: Use strong, unique credentials from .env.prod RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER} RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS} # Consider setting RABBITMQ_NODENAME for clarity # RABBITMQ_NODENAME: rabbit@kiwiq-prod # Use modern configuration approach instead of deprecated env vars RABBITMQ_CONFIG_FILE: /etc/rabbitmq/rabbitmq.conf # Persist data volumes: - rabbitmq_data:/var/lib/rabbitmq - ./docker/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro networks: - prod_network # Resource limits deploy: resources: limits: memory: 2G cpus: '1.0' reservations: memory: 1G cpus: '0.5' # Logging logging: driver: "json-file" options: max-size: "100m" max-file: "3" # No ports exposed externally (including management UI 15672) # Access management UI via SSH tunnel or internal network if needed. # Optional: Add basic healthcheck # healthcheck: # test: rabbitmq-diagnostics -q ping # interval: 30s # timeout: 10s # retries: 5 # Optional: Add production logging # Redis Cache/Store Service redis: # Using latest Redis version for production image: redis:latest container_name: kiwiq_prod_redis restart: unless-stopped # Needed for REDIS_PASSWORD env_file: - .env # Load password from .env.prod command: | redis-server --requirepass ${REDIS_PASSWORD} --maxmemory 256mb --maxmemory-policy allkeys-lru --save "" # Persist data volumes: - redis_data:/data networks: - prod_network # Resource limits deploy: resources: limits: memory: 512M # ⬇️ from 1.5G (saves 1GB) cpus: '0.25' # ⬇️ from 0.5 (saves 0.25) reservations: memory: 256M # ⬇️ from 512M cpus: '0.1' # ⬇️ from 0.25 # Logging logging: driver: "json-file" options: max-size: "50m" max-file: "3" # No ports exposed externally # Optional: Add basic healthcheck # healthcheck: # test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"] # interval: 10s # timeout: 5s # retries: 5 # Optional: Add production logging # Prefect Server Service (Orchestration API) prefect-server: # Use specific version tag (e.g., prefecthq/prefect:3.0.1-python3.12) image: prefecthq/prefect:3-latest container_name: kiwiq_prod_prefect_server restart: unless-stopped # Load Prefect specific variables env_file: - .env environment: # IMPORTANT: For production, consider using an external managed database (like RDS) # instead of the postgres container for Prefect's backend for better reliability/scalability. # If using the postgres container: PREFECT_API_DATABASE_CONNECTION_URL: "postgresql+asyncpg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${PREFECT_DB}" # If using external DB, ensure PREFECT_API_DATABASE_CONNECTION_URL is set correctly in .env.prod # Updated for subdomain approach (HTTPS via nginx + certbot) PREFECT_API_URL: "https://kq-prefect-server.prod.kiwiq.ai/api" # PREFECT_SERVER_UI_SERVE_BASE: "/" # Not needed for subdomain approach PREFECT_SERVER_API_AUTH_STRING: ${PREFECT_AUTH_STRING} PREFECT_SERVER_API_CSRF_PROTECTION_ENABLED: "true" # Temporarily disabled to test auth flow # Result Persistence (consider S3 or other external storage for production) PREFECT_TASKS_DEFAULT_PERSIST_RESULT: ${PREFECT_TASKS_DEFAULT_PERSIST_RESULT:-false} # PREFECT_RESULTS_DEFAULT_STORAGE_BLOCK: s3/your-prod-bucket # Example # ports: # - "4200:4200" # Commented out - nginx handles external access # Persist Prefect server state (if using internal storage/SQLite) volumes: - prefect_data:/root/.prefect depends_on: # postgres dependency removed - using managed PostgreSQL database app: condition: service_started networks: - prod_network entrypoint: ["prefect", "server", "start", "--host", "0.0.0.0", "--port", "4200"] # Resource limits deploy: resources: limits: memory: 2G cpus: '1.0' reservations: memory: 1G cpus: '0.5' # Logging logging: driver: "json-file" options: max-size: "100m" max-file: "3" # No ports exposed externally by default # Optional: Add production logging # Prefect Agent Service (Runs flows) prefect-agent: build: context: . # Use worker Dockerfile dockerfile: docker/Dockerfile.worker # Consider a separate Dockerfile.prod later. container_name: kiwiq_prod_prefect_agent restart: unless-stopped # Load necessary credentials and config (includes full DB/Service URLs) env_file: - .env depends_on: # Wait for the server API prefect-server: # Or add healthcheck to prefect-server and use service_healthy condition: service_started # Add dependencies based on what the agent/flows need (e.g., app, db, mq) # app: # Agent might need application code/utilities # condition: service_started weaviate: # condition: service_healthy condition: service_started # postgres dependency removed - using managed PostgreSQL database # mongo dependency removed - using managed MongoDB cluster rabbitmq: condition: service_started redis: condition: service_started app: condition: service_started environment: # PREFECT_API_URL: "${PREFECT_API_URL}" IS_PREFECT_WORKER: "true" PREFECT_LOGGING_LEVEL: INFO PREFECT_API_AUTH_STRING: ${PREFECT_AUTH_STRING} # Individual host vars are removed; rely on full URLs from .env.prod loaded above # POSTGRES_HOST: postgres # MONGO_HOST: mongo # RABBITMQ_HOST: rabbitmq # REDIS_HOST: redis # Ensure DATABASE_URL, MONGO_URL etc. are correctly set via .env.prod or here if needed directly # CodeRunner Docker-in-Docker configuration # Set HOST_PROJECT_PATH for proper Docker-in-Docker mounting in production # This tells CodeRunner how /app inside prefect-agent maps to the host filesystem HOST_PROJECT_PATH: "/home/ec2-user/stealth-backend" # In production, /app in container maps to /app on host # Resource limits (includes MongoDB and PostgreSQL resources from managed services migration) deploy: resources: limits: memory: 47.5G # 35.5G + 8G (mongo) + 4G (postgres) cpus: '12.5' # 8.0 + 2.5 (mongo) + 2.0 (postgres) reservations: memory: 34G # 26G + 5G (mongo) + 3G (postgres) cpus: '8.25' # 6.0 + 1.5 (mongo) + 0.75 (postgres) # Logging logging: driver: "json-file" options: max-size: "200m" max-file: "5" # Similar to app, mount code only if necessary and not baked into the image # Keep for now, consider removing for pure prod image volumes: - /var/run/docker.sock:/var/run/docker.sock - ./:/app # Command to start the agent polling a specific work pool and/or work queues # Replace 'default-agent-pool' and 'default' with your actual production work pool/queue name(s) # command: ["prefect", "agent", "start", "--pool", "default-agent-pool", "--work-queue", "default"] networks: - prod_network # Optional: Add production logging # Define named volumes for data persistence volumes: weaviate_data: driver: local # mongo_data and mongo_config removed - using managed MongoDB cluster # postgres_data removed - using managed PostgreSQL database rabbitmq_data: driver: local redis_data: driver: local prefect_data: driver: local # Removed volumes for Nginx/Certbot as they're now using bind mounts # Define the custom network for inter-service communication networks: # Default bridge network driver prod_network: driver: bridge