meta: title: "DeepSeek-V4-Flash" slug: "deepseek-v4-flash" provider: "DeepSeek" description: "DeepSeek V4 MoE model with hybrid CSA+HCA attention, manifold-constrained hyper-connections, and three-tier reasoning (Non-think / Think High / Think Max)." date_added: 2026-07-31 date_updated: 2026-08-14 difficulty: hard tasks: - text performance_headline: "Compact 284B/13B V4 sibling — single-node 1M-context serving with FP4+FP8 weights, MTP, and DSpark speculative decoding" related_recipes: [] hardware: h200: verified b200: verified gb200: verified b300: verified gb300: verified dgx_station_gb300: verified dgx_spark_gb10: verified rtx_pro_6000_8x: verified mi300x: unsupported mi325x: verified mi355x: verified model: model_id: "deepseek-ai/DeepSeek-V4-Flash" min_vllm_version: "0.20.0" docker_image: amd: "vllm/vllm-openai-rocm:nightly" architecture: moe parameter_count: "284B" active_parameters: "13B" context_length: 1048576 flashinfer_autotune: true base_args: - "--trust-remote-code" - "--kv-cache-dtype" - "fp8" - "--block-size" - "256" features: tool_calling: description: "Enable tool calling with DeepSeek V4 chat template support." args: - "--tokenizer-mode" - "deepseek_v4" - "--tool-call-parser" - "deepseek_v4" - "--enable-auto-tool-choice" reasoning: description: "Enable reasoning/thinking mode with the DeepSeek V4 reasoning parser." args: - "--reasoning-parser" - "deepseek_v4" - "--reasoning-config" - '{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}' spec_decoding: description: "Speculative decoding — pick a drafting method." # Mode = the --speculative-config method only (args); the served checkpoint is # owned by the Variant row. Default is MTP; the 0731 (default) and DSpark # checkpoints flip this to dspark via their `default_modes`. default_mode: mtp modes: mtp: label: "MTP" description: "Built-in Multi-Token Prediction, 2 draft tokens (1 on Hopper)." # The preview FP4+FP8 and NVFP4 checkpoints ship an MTP head only — their # config.json carries no dspark_* block, so dspark can't be drafted there. hardware: rtx_pro_6000_8x: unsupported variants: - fp8 - nvfp4 args: - "--speculative-config" - '{"method":"mtp","num_speculative_tokens":2}' hardware_overrides: hopper: args: - "--speculative-config" - '{"method":"mtp","num_speculative_tokens":1}' dspark: label: "DSpark" description: "DSpark drafting, 7 draft tokens, probabilistic sampling. Requires the DSpark or 0731 checkpoint (Variant row)." # Only the fused checkpoints carry the DSpark draft module — the 0731 # release (default) and the DSpark re-pack of the preview weights. hardware: rtx_pro_6000_8x: unsupported variants: - default - dspark args: - "--speculative-config" - '{"method":"dspark","num_speculative_tokens":7,"draft_sample_method":"probabilistic"}' hardware_overrides: dgx_spark_gb10: args: - "--speculative-config" - '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"B12X_MLA_SPARSE"}' opt_in_features: - spec_decoding variants: default: # Official 0731 release: new weights (not a re-pack of the preview), same # architecture and same fused DSpark draft module. model_id: "deepseek-ai/DeepSeek-V4-Flash-0731" label: "FP8 (0731)" precision: fp8 vram_minimum_gb: 200 description: "Official DeepSeek-V4-Flash release (2026-07-31) superseding the preview — substantially stronger agentic performance, with the DSpark draft module attached." min_vllm_version: "0.25.0" docker_image: "vllm/vllm-openai:v0.25.0" default_modes: spec_decoding: dspark hardware_overrides: dgx_spark_gb10: docker_image: "eugr/spark-vllm-b12x:latest" extra_args: - "--max-num-seqs" - "8" - "--max-num-batched-tokens" - "8192" - "--gpu-memory-utilization" - "0.85" - "--enable-prefix-caching" - "--default-chat-template-kwargs.thinking=true" - "--default-chat-template-kwargs.reasoning_effort=high" - "--load-format" - "instanttensor" - "--moe-backend" - "b12x" - "--linear-backend" - "b12x" - "--attention-backend" - "B12X_MLA_SPARSE" - "--max-cudagraph-capture-size" - "64" - "--compilation-config" - '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' extra_env: PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" CUTE_DSL_ARCH: "sm_121a" VLLM_USE_AOT_COMPILE: "1" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_USE_MEGA_AOT_ARTIFACT: "-1" VLLM_MEMORY_PROFILE_INCLUDE_ATTN: "1" VLLM_USE_FLASHINFER_SAMPLER: "1" VLLM_USE_B12X_WO_PROJECTION: "1" VLLM_USE_B12X_MHC: "1" VLLM_USE_B12X_FP8_GEMM: "1" VLLM_USE_B12X_MOE: "1" VLLM_USE_B12X_SPARSE_INDEXER: "1" VLLM_USE_V2_MODEL_RUNNER: "1" B12X_MLA_SM120_UNIFIED: "1" B12X_MOE_FORCE_A8: "1" GLOO_SOCKET_IFNAME: "$IFACE_NAME" MN_IF_NAME: "$IFACE_NAME" NCCL_IB_DISABLE: "0" NCCL_IB_HCA: "$IB_IF" NCCL_IGNORE_CPU_AFFINITY: "1" NCCL_SOCKET_IFNAME: "$IFACE_NAME" OMPI_MCA_btl_tcp_if_include: "$IFACE_NAME" TP_SOCKET_IFNAME: "$IFACE_NAME" UCX_NET_DEVICES: "$IFACE_NAME" blackwell: extra_args: - "--moe-backend" - "deep_gemm_mega_moe" fp8: # Marked "(Preview)" rather than left bare: with the 0731 release on the same # row, an unqualified FP8 pill reads as "the plain one" instead of "the # superseded one". Matches DeepSeek's own naming — their benchmark tables # say `DeepSeek-V4-Flash-0731` vs `DeepSeek-V4-Flash (Preview)`. label: "FP8 (Preview)" precision: fp8 vram_minimum_gb: 170 description: "Native FP4+FP8 mixed checkpoint (MoE experts FP4, remaining params FP8)" # deep_gemm_mega_moe is an FP8-only MoE kernel — the FP8 checkpoint adds it on # Blackwell; the NVFP4 variant below adds nothing (its experts can't use it). hardware_overrides: dgx_spark_gb10: docker_image: "eugr/spark-vllm-b12x:latest" extra_args: - "--max-num-seqs" - "8" - "--max-num-batched-tokens" - "8192" - "--gpu-memory-utilization" - "0.85" - "--enable-prefix-caching" - "--default-chat-template-kwargs.thinking=true" - "--default-chat-template-kwargs.reasoning_effort=high" - "--load-format" - "instanttensor" - "--moe-backend" - "b12x" - "--linear-backend" - "b12x" - "--attention-backend" - "B12X_MLA_SPARSE" - "--max-cudagraph-capture-size" - "64" - "--compilation-config" - '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' extra_env: PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" CUTE_DSL_ARCH: "sm_121a" VLLM_USE_AOT_COMPILE: "1" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_USE_MEGA_AOT_ARTIFACT: "-1" VLLM_MEMORY_PROFILE_INCLUDE_ATTN: "1" VLLM_USE_FLASHINFER_SAMPLER: "1" VLLM_USE_B12X_WO_PROJECTION: "1" VLLM_USE_B12X_MHC: "1" VLLM_USE_B12X_FP8_GEMM: "1" VLLM_USE_B12X_MOE: "1" VLLM_USE_B12X_SPARSE_INDEXER: "1" VLLM_USE_V2_MODEL_RUNNER: "1" B12X_MLA_SM120_UNIFIED: "1" B12X_MOE_FORCE_A8: "1" GLOO_SOCKET_IFNAME: "$IFACE_NAME" MN_IF_NAME: "$IFACE_NAME" NCCL_IB_DISABLE: "0" NCCL_IB_HCA: "$IB_IF" NCCL_IGNORE_CPU_AFFINITY: "1" NCCL_SOCKET_IFNAME: "$IFACE_NAME" OMPI_MCA_btl_tcp_if_include: "$IFACE_NAME" TP_SOCKET_IFNAME: "$IFACE_NAME" UCX_NET_DEVICES: "$IFACE_NAME" blackwell: extra_args: - "--moe-backend" - "deep_gemm_mega_moe" nvfp4: model_id: "nvidia/DeepSeek-V4-Flash-NVFP4" precision: nvfp4 vram_minimum_gb: 170 description: "NVIDIA modelopt NVFP4 checkpoint (MoE experts NVFP4; attention, shared experts, router head, and MTP stay FP8) for Blackwell GPUs." hardware_overrides: dgx_spark_gb10: docker_image: "eugr/spark-vllm-b12x:latest" extra_args: - "--max-num-seqs" - "8" - "--max-num-batched-tokens" - "8192" - "--gpu-memory-utilization" - "0.85" - "--enable-prefix-caching" - "--default-chat-template-kwargs.thinking=true" - "--default-chat-template-kwargs.reasoning_effort=high" - "--load-format" - "instanttensor" - "--moe-backend" - "b12x" - "--linear-backend" - "b12x" - "--attention-backend" - "B12X_MLA_SPARSE" - "--max-cudagraph-capture-size" - "64" - "--compilation-config" - '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' extra_env: PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" CUTE_DSL_ARCH: "sm_121a" VLLM_USE_AOT_COMPILE: "1" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_USE_MEGA_AOT_ARTIFACT: "-1" VLLM_MEMORY_PROFILE_INCLUDE_ATTN: "1" VLLM_USE_FLASHINFER_SAMPLER: "1" VLLM_USE_B12X_WO_PROJECTION: "1" VLLM_USE_B12X_MHC: "1" VLLM_USE_B12X_FP8_GEMM: "1" VLLM_USE_B12X_MOE: "1" VLLM_USE_B12X_SPARSE_INDEXER: "1" VLLM_USE_V2_MODEL_RUNNER: "1" B12X_MLA_SM120_UNIFIED: "1" B12X_MOE_FORCE_A8: "1" GLOO_SOCKET_IFNAME: "$IFACE_NAME" MN_IF_NAME: "$IFACE_NAME" NCCL_IB_DISABLE: "0" NCCL_IB_HCA: "$IB_IF" NCCL_IGNORE_CPU_AFFINITY: "1" NCCL_SOCKET_IFNAME: "$IFACE_NAME" OMPI_MCA_btl_tcp_if_include: "$IFACE_NAME" TP_SOCKET_IFNAME: "$IFACE_NAME" UCX_NET_DEVICES: "$IFACE_NAME" dspark: # Official fused DSpark checkpoint — the preview weights with the DSpark draft # module attached (config.json gains the dspark_* block). A distinct served # repo, so it lives on the Variant axis; selecting it defaults the # spec-decoding method to dspark (see features.spec_decoding.modes). model_id: "deepseek-ai/DeepSeek-V4-Flash-DSpark" label: "DSpark" precision: fp8 vram_minimum_gb: 200 description: "Preview FP4+FP8 weights with the DSpark draft module baked in — same base checkpoint as the default variant plus a fused speculative decoder." # DSpark drafting landed in 0.25.0; 0.26.0 is the first release where it also # works on the MI325X/MI355X pills this recipe advertises (ROCm enablement). min_vllm_version: "0.25.0" default_modes: spec_decoding: dspark hardware_overrides: dgx_spark_gb10: docker_image: "eugr/spark-vllm-b12x:latest" extra_args: - "--max-num-seqs" - "8" - "--max-num-batched-tokens" - "8192" - "--gpu-memory-utilization" - "0.85" - "--enable-prefix-caching" - "--default-chat-template-kwargs.thinking=true" - "--default-chat-template-kwargs.reasoning_effort=high" - "--load-format" - "instanttensor" - "--moe-backend" - "b12x" - "--linear-backend" - "b12x" - "--attention-backend" - "B12X_MLA_SPARSE" - "--max-cudagraph-capture-size" - "64" - "--compilation-config" - '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' extra_env: PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" CUTE_DSL_ARCH: "sm_121a" VLLM_USE_AOT_COMPILE: "1" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_USE_MEGA_AOT_ARTIFACT: "-1" VLLM_MEMORY_PROFILE_INCLUDE_ATTN: "1" VLLM_USE_FLASHINFER_SAMPLER: "1" VLLM_USE_B12X_WO_PROJECTION: "1" VLLM_USE_B12X_MHC: "1" VLLM_USE_B12X_FP8_GEMM: "1" VLLM_USE_B12X_MOE: "1" VLLM_USE_B12X_SPARSE_INDEXER: "1" VLLM_USE_V2_MODEL_RUNNER: "1" B12X_MLA_SM120_UNIFIED: "1" B12X_MOE_FORCE_A8: "1" GLOO_SOCKET_IFNAME: "$IFACE_NAME" MN_IF_NAME: "$IFACE_NAME" NCCL_IB_DISABLE: "0" NCCL_IB_HCA: "$IB_IF" NCCL_IGNORE_CPU_AFFINITY: "1" NCCL_SOCKET_IFNAME: "$IFACE_NAME" OMPI_MCA_btl_tcp_if_include: "$IFACE_NAME" TP_SOCKET_IFNAME: "$IFACE_NAME" UCX_NET_DEVICES: "$IFACE_NAME" blackwell: extra_args: - "--moe-backend" - "deep_gemm_mega_moe" compatible_strategies: - single_node_tp - single_node_tep - single_node_dep - multi_node_dep - pd_cluster # TP+EP is the default deployment; single_node_tp is offered only as a # latency-oriented alternative. default_strategy: single_node_tep kv_offload_support: offloading_cpu: verified offloading_fs: verified strategy_hardware: # Dual DGX Spark clusters need cross-node TP to shard the checkpoint across # the two GB10 nodes. Keep this opt-in exact to DGX Spark so larger Blackwell # platforms stay on their existing TEP/DEP/PD recommendations. multi_node_tp: dgx_spark_gb10: supported # PD splits into a prefill pool and a decode pool, each sized in whole nodes # (strategy_overrides.pd_cluster gives both `nodes: 1`), so the builder emits # a two-node deployment. RTX Pro 6000 8x is multi_node: false / scalable: # false in taxonomy.yaml — it can't be joined into a cluster, so that command # has nowhere to run. The JSON API already skips pd_cluster on non-scalable # hardware; this makes the Strategy row agree. multi_node_dep needs no entry: # the Nodes row is pinned to 1 here, which already hides every multi_node # strategy. pd_cluster: rtx_pro_6000_8x: unsupported hardware_overrides: blackwell: # Common to every variant on Blackwell. The FP8-only deep_gemm_mega_moe MoE # kernel is added per-variant (FP8 checkpoints), not here. extra_args: - "--attention_config.use_fp4_indexer_cache" - "True" amd: extra_args: - "--distributed-executor-backend" - "mp" - "--gpu-memory-utilization" - "0.9" - "--max-num-seqs" - "512" - "--max-num-batched-tokens" - "8192" - "--compilation-config" - '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}' extra_env: VLLM_ROCM_USE_AITER: "1" strategy_overrides: single_node_tp: # Latency-oriented TP-only deployment: no expert parallelism (single_node_tp # strategy doesn't add --enable-expert-parallel) and no MoE mega-kernel # backend on Blackwell. Autotune is disabled to minimize startup time. tp: 8 hardware_overrides: hopper: extra_args: - "--no-enable-flashinfer-autotune" blackwell: # Replaces recipe-level blackwell override: keeps the FP4 indexer cache # but drops --moe-backend deep_gemm_mega_moe (TP-only path uses the # default MoE backend). extra_args: - "--attention_config.use_fp4_indexer_cache" - "True" - "--no-enable-flashinfer-autotune" rtx_pro_6000_8x: extra_args: - "--attention_config.use_fp4_indexer_cache" - "False" single_node_tep: hardware_overrides: # sm_120 has neither Blackwell kernel: the FP4 indexer cache and # deep_gemm_mega_moe are SM100-only and abort at startup. Turned back off # here — these are the values vLLM resolves to unaided (FP8 indexer cache, # DEEPGEMM_MXFP4 MoE backend), which is the verified configuration. rtx_pro_6000_8x: extra_args: - "--attention_config.use_fp4_indexer_cache" - "False" - "--moe-backend" - "auto" single_node_dep: # Force DP=4 on all hardware (fills GB200's 4-GPU tray; on 8-GPU nodes use # half the GPUs per replica). dedupeArgs (last-wins) shadows the default # --data-parallel-size that command-synthesis emits from gpu_count. extra_args: - "--data-parallel-size" - "4" - "--compilation-config" - '{"cudagraph_mode":"FULL_AND_PIECEWISE", "custom_ops":["all"]}' hardware_overrides: # Same sm_120 opt-out as single_node_tep. rtx_pro_6000_8x: extra_args: - "--attention_config.use_fp4_indexer_cache" - "False" - "--moe-backend" - "auto" multi_node_dep: extra_args: - "--compilation-config" - '{"cudagraph_mode":"FULL_AND_PIECEWISE", "custom_ops":["all"]}' hardware_overrides: # Same sm_120 opt-out as single_node_tep. rtx_pro_6000_8x: extra_args: - "--attention_config.use_fp4_indexer_cache" - "False" - "--moe-backend" - "auto" multi_node_tp: hardware_overrides: # Multi-node TP is only enabled for DGX Spark in this recipe. Use a # generation-level strategy override so it replaces the generic Blackwell # FP4-indexer / deep_gemm_mega_moe overrides instead of layering on top. blackwell: extra_args: - "--moe-backend" - "b12x" - "--linear-backend" - "b12x" - "--attention-backend" - "B12X_MLA_SPARSE" pd_cluster: env: VLLM_USE_NCCL_SYMM_MEM: "1" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" prefill: nodes: 1 # 1 GB200 node = 4 GPUs parallelism: dep # DP + expert-parallel pool vllm_args: - "--enforce-eager" - "--max-num-seqs" - "8" - "--max-num-batched-tokens" - "65536" - "--no-disable-hybrid-kv-cache-manager" - "--enable-sleep-mode" env: {} decode: nodes: 1 # 1 GB200 nodes = 4 GPUs parallelism: dep vllm_args: - "--max-num-seqs" - "1536" - "--max-num-batched-tokens" - "1536" - "--max-cudagraph-capture-size" - "1536" - "--compilation-config" - '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - "--no-disable-hybrid-kv-cache-manager" - "--enable-sleep-mode" env: {} hardware_notes: dgx_spark_gb10: title: "DGX Spark setup required" body: "This hardware uses the community eugr/spark-vllm-b12x:latest docker image and a 2-node TP launch. Before running the generated command, check the DGX Spark cluster guide for IFACE_NAME, IB_IF and HEAD_IP." link: label: "DGX Spark cluster guide" href: "#dgx-spark-gb10-cluster" guide: | ## Overview DeepSeek-V4-Flash is a 284B-total / 13B-active MoE model in the V4 preview family. It pairs a **hybrid attention stack** — Compressed Sparse Attention (CSA) + Heavily Compressed Attention (HCA) — with **Manifold-Constrained Hyper-Connections (mHC)** to reach 27% of V3.2's per-token inference FLOPs and 10% of V3.2's KV cache at 1M context. Pre-trained on 32T+ tokens; post-training is a two-stage pipeline (domain- specific expert cultivation + unified consolidation via on-policy distillation). Checkpoint is **FP4+FP8 mixed**: MoE expert weights are stored in FP4 while the remaining (attention / norm / router) params stay in FP8. An **NVFP4 variant** (`nvidia/DeepSeek-V4-Flash-NVFP4`) is also available — NVIDIA modelopt re-quantizes the MoE experts to standard NVFP4 while attention, shared experts, router head, and MTP stay FP8. Pick it from the **Variant** row; it runs on Blackwell GPUs with the FP4 indexer cache. Unlike the native FP8 checkpoint, the NVFP4 experts don't support the `deep_gemm_mega_moe` MoE kernel (FP8-only), so it runs on the default MoE backend. ## Checkpoints Four checkpoints are on the **Variant** row. They differ in *which weights you serve* and *which draft module ships with them* — the speculative method itself is picked on the **Spec Decoding** row. | Variant | Repo | Draft | Notes | | :-- | :-- | :-- | :-- | | **FP8 (0731)** (default) | `deepseek-ai/DeepSeek-V4-Flash-0731` | DSpark | Official release, new weights + DSpark | | **FP8 (Preview)** | `deepseek-ai/DeepSeek-V4-Flash` | MTP | Preview FP4+FP8 mixed weights | | **NVFP4** | `nvidia/DeepSeek-V4-Flash-NVFP4` | MTP | modelopt re-quant, Blackwell | | **DSpark** | `deepseek-ai/DeepSeek-V4-Flash-DSpark` | DSpark | Preview weights + fused DSpark module | **0731 is the official DeepSeek-V4-Flash release** and the default here, superseding the preview with substantially stronger agentic capability — 82.7 on Terminal Bench 2.1 and 54.4 on DeepSWE, versus 61.8 and 7.3 for the preview. It beats DeepSeek-V4-Pro (Preview) on every agentic benchmark DeepSeek published despite its far smaller activated parameter count. The preview weights remain available as the **FP8 (Preview)** variant for reproducing earlier results. **DSpark** is *not* new weights — it is the preview checkpoint with a speculative decoding module attached (see [DeepSpec](https://github.com/deepseek-ai/DeepSpec)). Both fused checkpoints add a `dspark_*` block to `config.json`; the preview and NVFP4 checkpoints have no such block, which is why the DSpark method is offered only on the two variants that carry the draft. Selecting either one auto-enables **Spec Decoding** with `method: dspark`, emitting: ``` --speculative-config '{"method":"dspark","num_speculative_tokens":7,"draft_sample_method":"probabilistic"}' ``` The fused checkpoints are ~167 GB on disk versus ~160 GB for the preview — the draft module is the difference. Both require **vLLM 0.25.0**: DSpark drafting itself landed in 0.25.0, but ROCm support for it (MI325X / MI355X) only shipped in 0.26.0. On NVIDIA-only deployments 0.25.0 is sufficient. ## Reasoning modes The chat template exposes three reasoning-effort modes: - **Non-think** — fast, intuitive responses. - **Think High** — explicit chain-of-thought for logical analysis and planning. - **Think Max** — maximum reasoning effort; requires `--max-model-len >= 393216` (384K tokens) to avoid truncation. Recommended sampling: `temperature = 1.0`, `top_p = 1.0`. On the **0731** variant the effort levels are named `low` / `high` / `max`, and DeepSeek recommends `top_p = 0.95` for agentic scenarios (`1.0` otherwise) with `temperature = 1.0`. Allow up to **384K output tokens** at the `high` and `max` levels. Note that neither fused checkpoint ships a Jinja chat template — the repos provide an `encoding/` folder with `encode_messages` / `parse_message_from_completion_text` helpers instead. Serving through vLLM with `--tokenizer-mode deepseek_v4` (the **Tool Calling** pill) applies the built-in DeepSeek-V4 encoding, so the OpenAI- compatible endpoint works without the helper scripts. ### OpenAI Client Example For DeepSeek-V4, keep reasoning controls in `chat_template_kwargs`, as it exposes a custom **Think Max** mode via `"reasoning_effort": "max"`. ```python from openai import OpenAI client = OpenAI(base_url="http://localhost:8000/v1", api_key="EMPTY") model = "deepseek-ai/DeepSeek-V4-Flash" messages = [{"role": "user", "content": "What is 17*19? Return only the final integer."}] # Non-think resp = client.chat.completions.create( model=model, messages=messages, ) # Think High resp = client.chat.completions.create( model=model, messages=messages, extra_body={ "chat_template_kwargs": { "thinking": True, "reasoning_effort": "high", }, }, ) # Think Max resp = client.chat.completions.create( model=model, messages=messages, extra_body={ "chat_template_kwargs": { "thinking": True, "reasoning_effort": "max", }, }, ) ``` ## Recommended deployment Non-disaggregated serving on multi-GPU supported hardware: single-node DP + EP with `--data-parallel-size 4`. Fills a GB200 NVL4 tray exactly; uses 4 of 8 GPUs per replica on H200/B200/B300 (leaving headroom for throughput-vs-latency tuning). On DGX Station, use the single-GPU launch below. For disaggregated prefill/decode on GB200, use the PD Cluster tab. ### DGX Spark (GB10) cluster DGX Spark requires the community Spark vLLM build for this model rather than the stock vLLM release/nightly image. The **DGX Spark (GB10)** hardware pill pins the Docker tab to `eugr/spark-vllm-b12x:latest`, whose build carries the B12X backends and Spark-specific kernels for the GB10 / SM12x family. Use **2 nodes**. A single GB10's 128 GB unified memory is below the FP8/NVFP4 checkpoint footprint, so the recipe grants DGX Spark a Spark-only **Multi-Node TP** layout (`--tensor-parallel-size 2`, one GB10 per node). The generated command also sets the B12X runtime environment and the Spark network environment; export `IFACE_NAME`, `IB_IF` and `HEAD_IP` on each node before launching. To identify the values, first list the ConnectX Ethernet interfaces, RoCE devices and the IP address of the UP ConnectX interface on each Spark: ```bash $ ibdev2netdev roceP2p1s0f0 port 1 ==> enP2p1s0f0np0 (Up) roceP2p1s0f1 port 1 ==> enP2p1s0f1np1 (Down) rocep1s0f0 port 1 ==> enp1s0f0np0 (Up) rocep1s0f1 port 1 ==> enp1s0f1np1 (Down) $ ip addr show enp1s0f0np0 3: enp1s0f0np0: mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 4c:62:56:c1:83:68 brd ff:ff:ff:ff:ff:ff inet 100.100.100.10/24 brd 100.100.100.255 scope global noprefixroute enp1s0f0np0 valid_lft forever preferred_lft forever ``` From above example values, we can get the following common values: ```bash # Common values for all nodes HEAD_IP="100.100.100.10" # IP of the head node's UP ConnectX interface IFACE_NAME="enp1s0f0np0" # The head node's UP ConnectX Ethernet interface IB_IF="rocep1s0f0,roceP2p1s0f0" # Comma-separated RoCE device name list corresponding to the UP ConnectX interface ``` ### DGX Station Single-GPU ```bash vllm serve deepseek-ai/DeepSeek-V4-Flash \ --tensor-parallel-size 1 --pipeline-parallel-size 1 \ --kv-cache-dtype fp8 --trust-remote-code --block-size 256 \ --gpu-memory-utilization 0.92 \ --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ --attention_config.use_fp4_indexer_cache True \ --tokenizer-mode deepseek_v4 --tool-call-parser deepseek_v4 \ --enable-auto-tool-choice --reasoning-parser deepseek_v4 \ --max-cudagraph-capture-size 128 \ --speculative-config '{"method":"mtp","num_speculative_tokens":3}' ``` ### RTX PRO 6000 8× (8×96 GB, sm_120) Single-node TP8 + expert parallelism on the **0731** checkpoint. Verified with the `vllm/vllm-openai:v0.25.0` image on 8× RTX PRO 6000 (PCIe, no NVLink). Two host-side prerequisites before launching: ```bash # vLLM 0.25.0's DeepSeek-V4 sparse MLA decode passes `swa_topk_lens`, which # only exists in FlashInfer 0.6.14; the image pins 0.6.13. --no-deps is # required — a plain install pulls a different torch and breaks vLLM's # compiled extensions. /usr/bin/python3.12 -m pip install --no-deps flashinfer-python==0.6.14 # flashinfer-cubin has no matching 0.6.14 release, so skip the parity check. export FLASHINFER_DISABLE_VERSION_CHECK=1 ``` ```bash vllm serve deepseek-ai/DeepSeek-V4-Flash-0731 \ --trust-remote-code \ --kv-cache-dtype fp8 \ --block-size 256 \ --enable-expert-parallel \ --tensor-parallel-size 8 \ --tokenizer-mode deepseek_v4 \ --tool-call-parser deepseek_v4 \ --enable-auto-tool-choice \ --reasoning-parser deepseek_v4 ``` **Speculative decoding is unavailable on sm_120.** Both MTP and DSpark abort during warmup with `Check failed: num_tokens > 64` from `sparse_mla_sm120_paged_attention` — the SM120 sparse MLA kernel requires more than 64 tokens per batch, while speculative verification submits only the draft length (7 for DSpark). The Spec Decoding pill is disabled for this profile. ### MI325X (1×256GB) Validated end-to-end with vLLM 0.25.0 and PyTorch HIP 7.2.53211. Use this conservative TP1 eager-mode launch instead of the MI355X throughput settings: ```bash export VLLM_ROCM_USE_AITER=1 vllm serve deepseek-ai/DeepSeek-V4-Flash \ --host 0.0.0.0 \ --port 8002 \ --tensor-parallel-size 1 \ --kv-cache-dtype fp8_e4m3 \ --max-model-len 4096 \ --enable-chunked-prefill \ --max-num-batched-tokens 256 \ --kv-cache-memory-bytes 10000000000 \ --distributed-executor-backend mp \ --trust-remote-code \ --tokenizer-mode deepseek_v4 \ --moe-backend triton_unfused \ --enforce-eager ``` Chunked prefill allows 4K requests while scheduling at most 256 tokens per batch. This configuration loaded the 148.66 GiB checkpoint and passed model-list and chat probes in non-thinking and Think High modes during a 24-hour allocation. Only TP1 at 4K context is verified; TP2+ did not validate successfully. Think Max requires at least 384K context and is not available with this configuration. ### MI355X (4×288GB) If you want to further lower the latency, you can run with `--tensor-parallel-size 8`. ```bash export VLLM_ROCM_USE_AITER=1 vllm serve deepseek-ai/DeepSeek-V4-Flash \ --host localhost \ --port 8001 \ --dtype auto \ --kv-cache-dtype fp8 \ --tensor-parallel-size 4 \ --max-num-seqs 512 \ --max-num-batched-tokens 8192 \ --distributed-executor-backend mp \ --trust-remote-code \ --gpu-memory-utilization 0.9 \ --tokenizer-mode deepseek_v4 \ --reasoning-parser deepseek_v4 \ --tool-call-parser deepseek_v4 \ --enable-auto-tool-choice \ --compilation-config '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}' ``` #### MI355X is validated on GSM8K dataset: ##### Launch command ```bash MODEL=deepseek-ai/DeepSeek-V4-Flash lm_eval --model local-completions \ --model_args model=$MODEL,base_url=http://0.0.0.0:8001/v1/completions,num_concurrent=128,max_retries=10,max_gen_toks=2048,timeout=60000 \ --batch_size auto \ --tasks gsm8k \ --num_fewshot 8 \ --output_path . 2>&1 | tee -a eval.log ``` ##### Reported result ```text local-completions ({'model': 'deepseek-ai/DeepSeek-V4-Flash', 'base_url': 'http://0.0.0.0:8001/v1/completions', 'num_concurrent': 128, 'max_retries': 10, 'max_gen_toks': 2048, 'timeout': 60000}), gen_kwargs: ({}), limit: None, num_fewshot: 8, batch_size: auto |Tasks|Version| Filter |n-shot| Metric | |Value | |Stderr| |-----|------:|----------------|-----:|-----------|---|-----:|---|-----:| |gsm8k| 3|flexible-extract| 8|exact_match|↑ |0.9439|± |0.0063| | | |strict-match | 8|exact_match|↑ |0.9431|± |0.0064| ``` ### H200 Single-Node PD (Mooncake) Single-host disaggregated serving: 4 prefill GPUs + 4 decode GPUs on one 8-GPU H200 node, using MooncakeConnector over RDMA for KV cache transfer. **Prefill** (GPUs 0–3, port 8000): ```bash docker run --gpus all \ --privileged --ipc=host -p 8000:8000 \ --network host \ -v ~/.cache/huggingface:/root/.cache/huggingface \ -v /mnt/shared:/mnt/shared \ -e TILELANG_CLEANUP_TEMP_FILES=1 \ -e VLLM_DISABLE_COMPILE_CACHE=1 \ -e VLLM_ENGINE_READY_TIMEOUT_S=3600 \ -e VLLM_RPC_TIMEOUT=600000 \ -e VLLM_LOG_STATS_INTERVAL=1 \ -e VLLM_MOONCAKE_BOOTSTRAP_PORT=8998 \ -e CUDA_VISIBLE_DEVICES=0,1,2,3 \ vllm/vllm-openai:v0.25.0 \ deepseek-ai/DeepSeek-V4-Flash \ --trust-remote-code \ --kv-cache-dtype fp8 \ --block-size 256 \ --port 8000 \ --data-parallel-size 4 \ --enable-expert-parallel \ --tokenizer-mode deepseek_v4 \ --reasoning-parser deepseek_v4 \ --max-model-len auto \ --max-num-batched-tokens 16384 \ --max-num-seqs 8 \ --enforce-eager \ --no-disable-hybrid-kv-cache-manager \ --disable-uvicorn-access-log \ --kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"mooncake_protocol":"rdma"}}' ``` **Decode** (GPUs 4–7, port 8001): ```bash docker run --gpus all \ --privileged --ipc=host -p 8001:8001 \ --network host \ -v ~/.cache/huggingface:/root/.cache/huggingface \ -v /mnt/shared:/mnt/shared \ -e TILELANG_CLEANUP_TEMP_FILES=1 \ -e VLLM_DISABLE_COMPILE_CACHE=1 \ -e VLLM_ENGINE_READY_TIMEOUT_S=3600 \ -e VLLM_RPC_TIMEOUT=600000 \ -e VLLM_LOG_STATS_INTERVAL=1 \ -e VLLM_MOONCAKE_BOOTSTRAP_PORT=9889 \ -e CUDA_VISIBLE_DEVICES=4,5,6,7 \ vllm/vllm-openai:v0.25.0 \ deepseek-ai/DeepSeek-V4-Flash \ --trust-remote-code \ --kv-cache-dtype fp8 \ --block-size 256 \ --port 8001 \ --data-parallel-size 4 \ --enable-expert-parallel \ --tokenizer-mode deepseek_v4 \ --reasoning-parser deepseek_v4 \ --max-model-len auto \ --max-num-seqs 512 \ --max-num-batched-tokens 512 \ --compilation-config '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY","max_cudagraph_capture_size":512,"compile_ranges_endpoints":[512]}' \ --no-disable-hybrid-kv-cache-manager \ --disable-uvicorn-access-log \ --kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"mooncake_protocol":"rdma"}}' ``` **[Router](https://github.com/vllm-project/router)**: ```bash pip install vllm-router vllm-router --policy round_robin \ --vllm-pd-disaggregation \ --prefill http://localhost:8000 \ --decode http://localhost:8001 \ --host 127.0.0.1 \ --port 30000 \ --intra-node-data-parallel-size 4 \ --kv-connector mooncake ``` ### H200 Single-Node PD (Nixl) Single-host disaggregated serving: 4 prefill GPUs + 4 decode GPUs on one 8-GPU H200 node, using NixlConnector for KV cache transfer. **Prefill** (GPUs 0–3, port 8000): ```bash docker run --gpus all \ --privileged --ipc=host -p 8000:8000 \ --network host \ -v ~/.cache/huggingface:/root/.cache/huggingface \ -v /mnt/shared:/mnt/shared \ -e TILELANG_CLEANUP_TEMP_FILES=1 \ -e VLLM_DISABLE_COMPILE_CACHE=1 \ -e VLLM_ENGINE_READY_TIMEOUT_S=3600 \ -e VLLM_RPC_TIMEOUT=600000 \ -e VLLM_LOG_STATS_INTERVAL=1 \ -e VLLM_NIXL_SIDE_CHANNEL_PORT=5557 \ -e CUDA_VISIBLE_DEVICES=0,1,2,3 \ vllm/vllm-openai:v0.25.0 \ deepseek-ai/DeepSeek-V4-Flash \ --trust-remote-code \ --kv-cache-dtype fp8 \ --block-size 256 \ --port 8000 \ --data-parallel-size 4 \ --enable-expert-parallel \ --tokenizer-mode deepseek_v4 \ --reasoning-parser deepseek_v4 \ --max-model-len auto \ --max-num-batched-tokens 16384 \ --max-num-seqs 8 \ --enforce-eager \ --no-disable-hybrid-kv-cache-manager \ --disable-uvicorn-access-log \ --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' ``` **Decode** (GPUs 4–7, port 8001): ```bash docker run --gpus all \ --privileged --ipc=host -p 8001:8001 \ --network host \ -v ~/.cache/huggingface:/root/.cache/huggingface \ -v /mnt/shared:/mnt/shared \ -e TILELANG_CLEANUP_TEMP_FILES=1 \ -e VLLM_DISABLE_COMPILE_CACHE=1 \ -e VLLM_ENGINE_READY_TIMEOUT_S=3600 \ -e VLLM_RPC_TIMEOUT=600000 \ -e VLLM_LOG_STATS_INTERVAL=1 \ -e VLLM_NIXL_SIDE_CHANNEL_PORT=5558 \ -e CUDA_VISIBLE_DEVICES=4,5,6,7 \ vllm/vllm-openai:v0.25.0 \ deepseek-ai/DeepSeek-V4-Flash \ --trust-remote-code \ --kv-cache-dtype fp8 \ --block-size 256 \ --port 8001 \ --data-parallel-size 4 \ --enable-expert-parallel \ --tokenizer-mode deepseek_v4 \ --reasoning-parser deepseek_v4 \ --max-model-len auto \ --max-num-seqs 512 \ --max-num-batched-tokens 512 \ --compilation-config '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY","max_cudagraph_capture_size":512,"compile_ranges_endpoints":[512]}' \ --no-disable-hybrid-kv-cache-manager \ --disable-uvicorn-access-log \ --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' ``` **[Router](https://github.com/vllm-project/router)**: ```bash pip install vllm-router vllm-router --policy round_robin \ --vllm-pd-disaggregation \ --prefill http://localhost:8000 \ --decode http://localhost:8001 \ --host 127.0.0.1 \ --port 30000 \ --intra-node-data-parallel-size 4 \ --kv-connector nixl ```