name: glm53-flash-r35
services:
  download:
    image: localinferencelab/vllm@sha256:7a425c6864b951bbc368111490a4b0ac69d8cd0dd4987075b2c7d40b753b1bf5
    entrypoint: ["python", "/download.py"]
    environment:
      HF_TOKEN: ${HF_TOKEN:-}
      HF_HUB_OFFLINE: "0"
    volumes:
      - ${MODEL_DIR:-./models}:/models
      - ./scripts/download.py:/download.py:ro
    restart: "no"

  engine:
    image: localinferencelab/vllm@sha256:7a425c6864b951bbc368111490a4b0ac69d8cd0dd4987075b2c7d40b753b1bf5
    depends_on:
      download:
        condition: service_completed_successfully
    command: ["/model", "--mm-processor-device", "cpu", "--limit-mm-per-prompt", '{"image":64,"video":1}']
    environment:
      MODEL: "/model"
      DFLASH_MODEL: "/draft"
      MODEL_REVISION: "46aaae8a82032f77100f2f03e9cc11b391df3b4d"
      DFLASH_MODEL_REVISION: "dfa270d7eb8df37e0cd0d4420f8dd0bd24ffcd50"
      CACHE_MODE: "vram"
      KV_CACHE_QUANT: "fp8_ds_mla"
      TP: "4"
      DCP: "1"
      MAX_MODEL_LEN: "400000"
      MAX_NUM_SEQS: "8"
      MAX_NUM_BATCHED_TOKENS: "4096"
      PREFILL_SCHEDULE_INTERVAL: "1"
      PREFILL_COMPUTE_SHARE: "0.4"
      GPU_MEMORY_UTILIZATION: "0.93"
      SPECULATOR: "dflash2"
      DFLASH_DEPTH: "7"
      SERVED_MODEL_NAME: "glm-5.3-flash"
      PORT: "8000"
      CUDA_DEVICE_ORDER: "PCI_BUS_ID"
      HF_HUB_OFFLINE: "1"
      MAX_CUDAGRAPH_CAPTURE_SIZE: "64"
      CUDAGRAPH_CAPTURE_SIZES: "1 2 4 8 16 32 40 48 64"
    volumes:
      - ${MODEL_DIR:-./models}/target:/model:ro
      - ${MODEL_DIR:-./models}/draft:/draft:ro
      - compile-cache:/root/.cache
    shm_size: 16gb
    init: true
    restart: unless-stopped
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["0", "1", "2", "3"]
              capabilities: [gpu]
    healthcheck:
      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
      interval: 15s
      timeout: 6s
      start_period: 15m
      retries: 4

  api:
    build: ./gateway
    environment:
      API_KEY: ${API_KEY:?Set API_KEY in .env}
      UPSTREAM: http://engine:8000
      HISTORY_IMAGE_LIMIT: "8"
    depends_on:
      engine:
        condition: service_healthy
    ports:
      - "${BIND_ADDRESS:-127.0.0.1}:${PORT:-8000}:8080"
    restart: unless-stopped
    read_only: true
    cap_drop: [ALL]
    security_opt: ["no-new-privileges:true"]
    healthcheck:
      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health', timeout=5)"]
      interval: 30s
      timeout: 6s
      retries: 3
volumes:
  compile-cache: