services:
  vllm:
    build: .
    image: vllm-spark:qwen3.5-nvfp4
    container_name: vllm-txn545-qwen3.5-122b-nvfp4
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - VLLM_USAGE_SOURCE=production
      # BF16 matmul precision (NVFP4 dequant 후 BF16 연산)
      - TORCH_MATMUL_PRECISION=high
      # CUDA 할당기 단편화 억제 (장시간 추론 안정성)
      - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
      # NVFP4 MoE:
      # VLLM_USE_FLASHINFER_MOE_FP4=1 → flashinfer-cutlass 사용 시도
      #   - latency: TRT-LLM JIT (SM100/10.x 전용) → SM12x 실패
      #   - throughput: SM120 MXFP4_MINIMAL → FP8 activations 필요, BF16 전달 시 실패
      # VLLM_USE_FLASHINFER_MOE_FP4=0 (기본값) → native cutlass_moe_fp4 사용 → SM12x 정상 동작
      # [Experimental] Force Marlin W4A16 MoE backend instead of CUTLASS W4A4.
      # Avarok saw +13% on their build; on SM121 with native CUTLASS FP4 it may be slower.
      # Set to "1" to enable; "0" (default) keeps native CUTLASS.
      - VLLM_NVFP4_MOE_FORCE_MARLIN=0
      # A/B tested: Marlin W4A16 ≈ CUTLASS W4A4 on SM121 122B (~15.2 tok/s both).
      # No benefit from Marlin; keep native CUTLASS for stability.
      # Disable FlashInfer MoE FP4 backends (FLASHINFER_CUTLASS requires FP8 activations
      # from the profile run dummy data; use VLLM native CUTLASS MoE instead).
      - VLLM_USE_FLASHINFER_MOE_FP4=0
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    volumes:
      - ${MODEL_HOST_PATH}:/models/vLLM_txn545_Qwen3.5-122B-A10B-NVFP4:ro
      - ./.cache/flashinfer:/root/.cache/flashinfer
      - ./.cache/vllm:/root/.cache/vllm
    ports:
      - "${HOST_PORT}:8000"
    ipc: host
    ulimits:
      memlock:
        soft: -1
        hard: -1
    # MTP speculative decoding: ENABLED (BF16 mtp.* weights extracted from original
    # Qwen/Qwen3.5-122B-A10B and merged into NVFP4 checkpoint on Mar 1 2026).
    # Dockerfile mtp_quant_exclusion_fix ensures MTP layers run in BF16 (not NVFP4).
    # --no-enable-chunked-prefill: required for MTP compatibility.
    command: >
      vllm serve /models/vLLM_txn545_Qwen3.5-122B-A10B-NVFP4
      --served-model-name vLLM_txn545_Qwen3.5-122B-A10B-NVFP4
      --tensor-parallel-size 1
      --max-model-len ${MAX_MODEL_LEN}
      --max-num-seqs ${MAX_NUM_SEQS}
      --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION}
      --trust-remote-code
      --host 0.0.0.0
      --port 8000
      --dtype auto
      --kv-cache-dtype fp8
      --attention-backend flashinfer
      --enable-chunked-prefill
      --max-num-batched-tokens ${MAX_NUM_BATCHED_TOKENS}
      --load-format safetensors
      --enable-prefix-caching
      --reasoning-parser qwen3
      --no-enable-flashinfer-autotune
      --speculative-config '{"method":"mtp","num_speculative_tokens":1}'
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:8000/health || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 10
      start_period: 600s
    restart: unless-stopped
    networks:
      - monitoring

networks:
  monitoring:
    external: true