# Adapted from NVIDIA Local AI Inference Recipes:
# https://nvidia.github.io/local-ai-inference-recipes/hardware/dgx-spark-2gpu/index.html?model=qwen%2Fqwen3.8-flash-next&os=linux&backend=vllm&memory=64&precision=fp4-local&memoryConfig=64
# Pull the named image on the head before --setup. --no-build copies it without replacing it.
# launch-cluster.sh adds the node count, node ranks, rendezvous address, and --headless.
name: Qwen3.8 Flash-Next on two 64GB DGX Sparks
recipe_version: '1'
model: nvidia/Qwen3.8-Flash-Next-FP4-Local
container: eugr/spark-vllm-b12x:nightly-20261015
cluster_only: true
build_args:
- --no-build
env:
  VLLM_PLE_TABLE_MEMORY: disk
  CUTE_DSL_ARCH: sm_121a
  SAFETENSORS_FAST_GPU: '1'
  VLLM_WORKER_MULTIPROC_METHOD: spawn
  VLLM_SSM_CONV_STATE_LAYOUT: DS
  VLLM_USE_AOT_COMPILE: '1'
  VLLM_USE_MEGA_AOT_ARTIFACT: '1'
  VLLM_USE_V2_MODEL_RUNNER: '1'
  B12X_POLICY_MODE: auto
  VLLM_ENABLE_ROCE_ALLREDUCE: '1'
  VLLM_ROCE_ALLREDUCE_MAX_SIZE: 2MB
  NCCL_BUFFSIZE: '1048576'
  NCCL_MAX_CTAS: '8'
  B12X_WEIGHTS_COMPILE_WORKERS: '4'
  B12X_STATE_COMPILE_WORKERS: '4'
  B12X_BIND_COMPILE_WORKERS: '4'
  VLLM_B12X_MOE_FP4_FORCE_A16: '1'
  B12X_W4A16_A4_PREFILL: '1'
defaults:
  host: 0.0.0.0
  port: 8000
  tensor_parallel: 2
  gpu_memory_utilization: 0.8
  max_model_len: 262144
command: |
  vllm serve nvidia/Qwen3.8-Flash-Next-FP4-Local \
    --host {host} \
    --port {port} \
    --trust-remote-code \
    --tensor-parallel-size {tensor_parallel} \
    --mamba-cache-mode align \
    --enable-prefix-caching \
    --enable-chunked-prefill \
    --dtype bfloat16 \
    --kv-cache-dtype fp8 \
    --quantization modelopt_mixed \
    --block-size 16 \
    --load-format b12x \
    --max-model-len {max_model_len} \
    --max-num-seqs 8 \
    --max-num-batched-tokens 2048 \
    --speculative-config '{{"method":"mtp","num_speculative_tokens":4}}' \
    --gdn-decode-kernel b12x \
    --linear-backend b12x \
    --moe-backend b12x \
    --no-enable-flashinfer-autotune \
    --mm-encoder-tp-mode data \
    --reasoning-parser qwen3 \
    --tool-call-parser qwen3_xml \
    --enable-auto-tool-choice \
    --compilation-config '{{"pass_config":{{"fuse_act_quant":true}},"max_cudagraph_capture_size":40,"cudagraph_capture_sizes":[1,2,4,5,8,10,16,20,24,32,40]}}' \
    --mm-processor-cache-gb 0 \
    --limit-mm-per-prompt '{{"image":1}}' \
    --gpu-memory-utilization {gpu_memory_utilization} \
    --kv-cache-memory-bytes 3G