{ "schema_version": 1, "generated": "DO NOT EDIT. Generated by build-release from the filtered public catalog.", "hardware": [ { "id": "dgx-spark", "display_name": "DGX Spark", "vendor": "nvidia", "kind": "sku", "architecture": "blackwell", "gpu_count": 1 }, { "id": "dgx-spark-2gpu", "display_name": "2× DGX Spark", "vendor": "nvidia", "kind": "configuration", "architecture": "blackwell", "gpu_count": 2, "base_hardware": "dgx-spark" }, { "id": "dgx-spark-4gpu", "display_name": "4× DGX Spark", "vendor": "nvidia", "kind": "configuration", "architecture": "blackwell", "gpu_count": 4, "base_hardware": "dgx-spark" }, { "id": "dgx-station", "display_name": "DGX Station", "vendor": "nvidia", "kind": "sku", "architecture": "blackwell", "gpu_count": 1 }, { "id": "dgx-station-2gpu", "display_name": "2× DGX Station", "vendor": "nvidia", "kind": "configuration", "architecture": "blackwell", "gpu_count": 2, "base_hardware": "dgx-station" }, { "id": "geforce-rtx", "display_name": "GeForce RTX GPUs", "vendor": "nvidia", "kind": "sku", "architecture": "blackwell", "gpu_count": 1 }, { "id": "rtx-pro", "display_name": "RTX PRO GPUs", "vendor": "nvidia", "kind": "sku", "architecture": "blackwell", "gpu_count": 1 }, { "id": "rtx-pro-2gpu", "display_name": "2× RTX PRO GPUs", "vendor": "nvidia", "kind": "configuration", "architecture": "blackwell", "gpu_count": 2, "base_hardware": "rtx-pro" }, { "id": "rtx-pro-4gpu", "display_name": "4× RTX PRO GPUs", "vendor": "nvidia", "kind": "configuration", "architecture": "blackwell", "gpu_count": 4, "base_hardware": "rtx-pro" }, { "id": "rtx-spark", "display_name": "RTX Spark", "vendor": "nvidia", "kind": "sku", "architecture": "grace-blackwell", "gpu_count": 1, "notes": "GB10 Grace Blackwell Superchip. llama.cpp runs natively on Windows.\nLinux is supported only via WSL2 (not yet modeled as its own platform here)." } ], "backends": [ { "id": "llama-cpp", "display_name": "llama.cpp", "protocol": "openai-compatible", "upstream": "https://github.com/ggml-org/llama.cpp", "notes": "Served with llama-server, which exposes an OpenAI-compatible API. Recipes use GGUF checkpoints and name the exact GGUF file; split checkpoints point to the first shard. Native launch recipes are supported on Linux and Windows." }, { "id": "vllm", "display_name": "vLLM", "protocol": "openai-compatible", "upstream": "https://github.com/vllm-project/vllm", "container_default": "vllm/vllm-openai:v0.24.0-ubuntu2404", "notes": "Recipes are self-contained: each declares its selected image and complete\nDocker startup and model-specific launch commands. Multi-node recipes include\nevery node role; source-supplied alternatives remain separate complete\noptions. Use the fallback default only when the source supplies neither an\nimage nor a startup command.\n\nKnown images: [vllm/vllm-openai:v0.24.0-ubuntu2404](https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.24.0-ubuntu2404)\n(fallback default); [vllm/vllm-openai:v0.25.1-ubuntu2404](https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.25.1-ubuntu2404);\n[vllm/vllm-openai:nightly](https://hub.docker.com/r/vllm/vllm-openai/tags?name=nightly)\n(mutable); [vllm/vllm-openai:nightly-311b3513af33bc29b4acb2fde2e9313e5e9966a0](https://hub.docker.com/r/vllm/vllm-openai/tags?name=nightly-311b3513af33bc29b4acb2fde2e9313e5e9966a0);\n[vllm/vllm-openai:nightly-aa9903490c616dc6871e5acc62cec7bb1e5e9434](https://hub.docker.com/r/vllm/vllm-openai/tags?name=nightly-aa9903490c616dc6871e5acc62cec7bb1e5e9434);\n[eugr/spark-vllm:latest](https://hub.docker.com/r/eugr/spark-vllm/tags?name=latest)\n(community, ARM64-only, mutable); [eugr/spark-vllm:nightly-20260704](https://hub.docker.com/r/eugr/spark-vllm/tags?name=nightly-20260704)\n(community, ARM64-only); [eugr/spark-vllm-b12x:latest](https://hub.docker.com/r/eugr/spark-vllm-b12x/tags?name=latest)\n(community, ARM64-only, mutable, custom B12x); [nvcr.io/nvidia/vllm:26.05.post1-py3](https://catalog.ngc.nvidia.com/orgs/nvidia/-/containers/vllm/-/tags)\n(NVIDIA, AMD64 and ARM64);\n\nEvaluation only: host networking exposes vLLM on host interfaces. Use a\nfirewall or reverse proxy, a scoped read-only HF token, and an isolated cache;\npin tested image, model, and code revisions for production." } ], "models": [ { "id": "deepseek/deepseek-v4-flash", "display_name": "DeepSeek-V4-Flash", "builder": "deepseek", "family": "DeepSeek-V4", "summary": "DeepSeek V4 Flash text-generation model.", "upstream": null, "modalities": [ "text" ], "recipes": [ { "id": "deepseek/deepseek-v4-flash/dgx-spark-2gpu/linux/vllm/fp4-fp8-mixed/mtp", "hardware": "dgx-spark-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.32.0rc1.dev301+g1ac942555.d20261016", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "fp4-fp8-mixed", "variant": "mtp", "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "target_label": "deepseek-ai/DeepSeek-V4-Flash", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× DGX Spark; one node rank per Spark; tensor parallel 2", "gpu_count": 2, "node_count": 2, "tensor_parallel_size": 2 }, "runtime": { "kind": "container", "label": "Container (community, mutable, ARM64)", "image": "eugr/spark-vllm:latest", "image_url": "https://hub.docker.com/r/eugr/spark-vllm/tags?name=latest", "setup_label": "Start container", "setup_target": "all-nodes", "setup_instructions": "Run on every node after setting `HF_TOKEN`; replace the node-specific placeholders.", "setup_command": "# Example two-Spark values (adjust to your network):\n# HEAD_IP=\"10.100.32.1\" # head node IP on the ETH subnet\n# ETH_IF=\"enp1s0f0np0\" # ConnectX baseboard ethernet (socket/control)\n# IB_IF=\"rocep1s0f0,roceP2p1s0f0\" # RoCE HCA names (data plane) - use both \"twins\"!\nexport HEAD_IP=\"\"\nexport ETH_IF=\"\"\nexport IB_IF=\",\"\nexport MASTER_PORT=29501\nexport NODE_RANK=\"<0-1>\"\nexport IMAGE=\"eugr/spark-vllm:latest\"\nexport HF_CACHE_DIR=\"$HOME/.cache/huggingface\"\n\nmkdir -p \"$HF_CACHE_DIR\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\"\n\ndocker run --rm -it \\\n --privileged \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit nofile=1048576:1048576 \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e VLLM_FLOAT32_MATMUL_PRECISION=high \\\n -e NODE_RANK=\"$NODE_RANK\" \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HF_CACHE_DIR:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n --entrypoint /bin/bash \\\n \"$IMAGE\"" }, "serve": { "label": "Launch head", "target": "head", "instructions": null, "command": "export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport VLLM_ALLOW_LONG_MAX_MODEL_LEN=\"1\"\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=\"1\"\n\nvllm serve deepseek-ai/DeepSeek-V4-Flash \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --tensor-parallel-size 2 \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --max-model-len 500000 \\\n --max-num-seqs 4 \\\n --max-num-batched-tokens 8192 \\\n --gpu-memory-utilization 0.85 \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":2}' \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --reasoning-config '{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}' \\\n --default-chat-template-kwargs.thinking=true \\\n --default-chat-template-kwargs.reasoning_effort=high \\\n --load-format instanttensor \\\n --nnodes 2 \\\n --node-rank 0 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\"", "port": 8000, "api_model": "deepseek-ai/DeepSeek-V4-Flash", "additional_commands": [ { "id": "worker", "label": "Launch worker", "target": "workers", "instructions": null, "command": "export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport VLLM_ALLOW_LONG_MAX_MODEL_LEN=\"1\"\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=\"1\"\n\nvllm serve deepseek-ai/DeepSeek-V4-Flash \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --tensor-parallel-size 2 \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --max-model-len 500000 \\\n --max-num-seqs 4 \\\n --max-num-batched-tokens 8192 \\\n --gpu-memory-utilization 0.85 \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":2}' \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --reasoning-config '{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}' \\\n --default-chat-template-kwargs.thinking=true \\\n --default-chat-template-kwargs.reasoning_effort=high \\\n --load-format instanttensor \\\n --nnodes 2 \\\n --node-rank 1 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\" \\\n --headless" } ] }, "launch": null, "sampling": null, "notes": "Use the same CX7 port on every node; assign both port subinterfaces on separate subnets; cache the image and checkpoint on every node; start workers before the head. Docker runs privileged.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"deepseek-ai/DeepSeek-V4-Flash\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"deepseek-ai/DeepSeek-V4-Flash\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "deepseek/deepseek-v4-flash-0731", "display_name": "DeepSeek-V4-Flash-0731", "builder": "deepseek", "family": "DeepSeek-V4", "summary": "DeepSeek V4 Flash 0731 inference configurations.", "upstream": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731", "modalities": [ "text" ], "recipes": [ { "id": "deepseek/deepseek-v4-flash-0731/dgx-spark-2gpu/linux/vllm/fp4-fp8-mixed", "hardware": "dgx-spark-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21830+g8244d1c44.d20261015", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "fp4-fp8-mixed", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [ "The source requires the custom B12x container image." ], "checkpoint": { "target": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731", "target_label": "deepseek-ai/DeepSeek-V4-Flash-0731", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× DGX Spark; one node rank per Spark; tensor parallel 2", "gpu_count": 2, "node_count": 2, "tensor_parallel_size": 2 }, "runtime": { "kind": "container", "label": "Container (community, mutable, ARM64, custom B12x)", "image": "eugr/spark-vllm-b12x:latest", "image_url": "https://hub.docker.com/r/eugr/spark-vllm-b12x/tags?name=latest", "setup_label": "Start container", "setup_target": "all-nodes", "setup_instructions": "Run on every node after setting HF_TOKEN; replace the node-specific placeholders.", "setup_command": "# Example two-Spark values (adjust to your network):\n# HEAD_IP=\"10.100.32.1\" # head node IP on the ETH subnet\n# ETH_IF=\"enp1s0f0np0\" # ConnectX baseboard ethernet (socket/control)\n# IB_IF=\"rocep1s0f0,roceP2p1s0f0\" # RoCE HCA names (data plane) - use both \"twins\"!\nexport HEAD_IP=\"\"\nexport ETH_IF=\"\"\nexport IB_IF=\",\"\nexport MASTER_PORT=29501\nexport IMAGE=\"eugr/spark-vllm-b12x:latest\"\nexport HF_CACHE_DIR=\"$HOME/.cache/huggingface\"\n\nmkdir -p \"$HF_CACHE_DIR\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\" \"$HOME/.cache/b12x\"\n\ndocker run --rm -it \\\n --privileged \\\n --security-opt seccomp=unconfined \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit nofile=1048576:1048576 \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HF_CACHE_DIR:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n -v $HOME/.cache/b12x:/root/.cache/b12x \\\n --entrypoint /bin/bash \\\n \"$IMAGE\"" }, "serve": { "label": "Launch head", "target": "head", "instructions": null, "command": "export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport CUTE_DSL_ARCH=\"sm_121a\"\nexport VLLM_USE_AOT_COMPILE=\"1\"\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=\"0\"\nexport VLLM_MEMORY_PROFILE_INCLUDE_ATTN=\"1\"\nexport VLLM_USE_FLASHINFER_SAMPLER=\"1\"\nexport VLLM_USE_B12X_WO_PROJECTION=\"1\"\nexport VLLM_USE_B12X_MHC=\"1\"\nexport VLLM_USE_B12X_FP8_GEMM=\"1\"\nexport VLLM_USE_B12X_MOE=\"1\"\nexport VLLM_USE_B12X_SPARSE_INDEXER=\"1\"\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\nexport B12X_MLA_SM120_UNIFIED=\"1\"\nexport B12X_MOE_FORCE_A8=\"1\"\nexport VLLM_MOE_SKIP_PADDING=\"0\" # 8/17 - need this due to latest changes in vLLM\nvllm serve deepseek-ai/DeepSeek-V4-Flash-0731 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --tensor-parallel-size 2 \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --max-model-len auto \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --gpu-memory-utilization 0.85 \\\n --enable-prefix-caching \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --reasoning-config '{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}' \\\n --default-chat-template-kwargs.thinking=true \\\n --default-chat-template-kwargs.reasoning_effort=high \\\n --load-format b12x \\\n --moe-backend b12x \\\n --linear-backend b12x \\\n --attention-backend B12X \\\n --max-cudagraph-capture-size 48 \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}' \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":5,\"draft_sample_method\":\"probabilistic\",\"attention_backend\":\"B12X\"}' \\\n --nnodes 2 \\\n --node-rank 0 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\"", "port": 8000, "api_model": "deepseek-ai/DeepSeek-V4-Flash-0731", "additional_commands": [ { "id": "worker", "label": "Launch worker", "target": "workers", "instructions": null, "command": "export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport CUTE_DSL_ARCH=\"sm_121a\"\nexport VLLM_USE_AOT_COMPILE=\"1\"\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=\"0\"\nexport VLLM_MEMORY_PROFILE_INCLUDE_ATTN=\"1\"\nexport VLLM_USE_FLASHINFER_SAMPLER=\"1\"\nexport VLLM_USE_B12X_WO_PROJECTION=\"1\"\nexport VLLM_USE_B12X_MHC=\"1\"\nexport VLLM_USE_B12X_FP8_GEMM=\"1\"\nexport VLLM_USE_B12X_MOE=\"1\"\nexport VLLM_USE_B12X_SPARSE_INDEXER=\"1\"\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\nexport B12X_MLA_SM120_UNIFIED=\"1\"\nexport B12X_MOE_FORCE_A8=\"1\"\nexport VLLM_MOE_SKIP_PADDING=\"0\" # 8/17 - need this due to latest changes in vLLM\nvllm serve deepseek-ai/DeepSeek-V4-Flash-0731 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --tensor-parallel-size 2 \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --max-model-len auto \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --gpu-memory-utilization 0.85 \\\n --enable-prefix-caching \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --reasoning-config '{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}' \\\n --default-chat-template-kwargs.thinking=true \\\n --default-chat-template-kwargs.reasoning_effort=high \\\n --load-format b12x \\\n --moe-backend b12x \\\n --linear-backend b12x \\\n --attention-backend B12X \\\n --max-cudagraph-capture-size 48 \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}' \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":5,\"draft_sample_method\":\"probabilistic\",\"attention_backend\":\"B12X\"}' \\\n --nnodes 2 \\\n --node-rank 1 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\" \\\n --headless" } ] }, "launch": null, "sampling": null, "notes": "Use the same CX7 port on every node; assign both port subinterfaces on separate subnets; cache the image and checkpoint on every node; start the worker before the head. Docker runs privileged.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"deepseek-ai/DeepSeek-V4-Flash-0731\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"deepseek-ai/DeepSeek-V4-Flash-0731\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "deepseek/deepseek-v4-flash-0731/dgx-spark-4gpu/linux/vllm/64gb/fp4-fp8-mixed", "hardware": "dgx-spark-4gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21830+g8244d1c44.d20261015", "backend_min_version": null, "min_vidmem_gb": 64, "precision": "fp4-fp8-mixed", "variant": null, "status": "optimized", "prerequisites": [ "Four DGX Spark nodes with at least 64 GB of memory per node and a configured RoCE network." ], "limitations": [ "The source requires the custom B12x container image." ], "checkpoint": { "target": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731", "target_label": "deepseek-ai/DeepSeek-V4-Flash-0731", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "4× DGX Spark; 64 GB per node; one node rank per Spark; tensor parallel 4", "gpu_count": 4, "node_count": 4, "tensor_parallel_size": 4 }, "runtime": { "kind": "container", "label": "Container (community, mutable, ARM64, custom B12x)", "image": "eugr/spark-vllm-b12x:latest", "image_url": "https://hub.docker.com/r/eugr/spark-vllm-b12x/tags?name=latest", "setup_label": "Start container", "setup_target": "all-nodes", "setup_instructions": "Run on every node after setting `HF_TOKEN`; replace the node-specific placeholders.", "setup_command": "# Example four-Spark values (adjust to your network):\n# HEAD_IP=\"10.100.32.1\" # head node IP on the ETH subnet\n# ETH_IF=\"enp1s0f0np0\" # ConnectX baseboard ethernet (socket/control)\n# IB_IF=\"rocep1s0f0,roceP2p1s0f0\" # RoCE HCA names (data plane) - use both \"twins\"!\nexport HEAD_IP=\"\"\nexport ETH_IF=\"\"\nexport IB_IF=\",\"\nexport MASTER_PORT=29501\nexport IMAGE=\"eugr/spark-vllm-b12x:latest\"\nexport HF_CACHE_DIR=\"$HOME/.cache/huggingface\"\n\nmkdir -p \"$HF_CACHE_DIR\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\" \"$HOME/.cache/b12x\"\n\ndocker run --rm -it \\\n --privileged \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit nofile=1048576:1048576 \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \\\n -e NCCL_BUFFSIZE=1048576 \\\n -e NCCL_MAX_CTAS=8 \\\n -e VLLM_ENABLE_ROCE_ALLREDUCE=1 \\\n -e VLLM_ROCE_ALLREDUCE_MAX_SIZE=2MB \\\n -e MIMALLOC_PURGE_DELAY=0 \\\n -e B12X_COMPILE_WORKERS=4 \\\n -e VLLM_USE_RUST_FRONTEND=1 \\\n -e MAX_JOBS=4 \\\n -e CMAKE_BUILD_PARALLEL_LEVEL=4 \\\n -e NINJAFLAGS=-j4 \\\n -e MAKEFLAGS=-j4 \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HF_CACHE_DIR:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n -v $HOME/.cache/b12x:/root/.cache/b12x \\\n --entrypoint /bin/bash \\\n \"$IMAGE\"" }, "serve": { "label": "Launch head", "target": "head", "instructions": "Run inside the head container after starting all three workers.", "command": "export CUTE_DSL_ARCH=\"sm_121a\"\nexport VLLM_USE_AOT_COMPILE=\"1\"\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=\"0\"\nexport VLLM_USE_MEGA_AOT_ARTIFACT=\"1\"\nexport VLLM_MEMORY_PROFILE_INCLUDE_ATTN=\"1\"\nexport VLLM_USE_FLASHINFER_SAMPLER=\"1\"\nexport VLLM_USE_B12X_WO_PROJECTION=\"1\"\nexport VLLM_USE_B12X_MHC=\"1\"\nexport VLLM_USE_B12X_FP8_GEMM=\"1\"\nexport VLLM_USE_B12X_MOE=\"1\"\nexport VLLM_USE_B12X_SPARSE_INDEXER=\"1\"\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\nexport VLLM_MOE_SKIP_PADDING=\"0\"\nexport B12X_MLA_SM120_UNIFIED=\"1\"\nexport B12X_MOE_FORCE_A8=\"1\"\n\nvllm serve deepseek-ai/DeepSeek-V4-Flash-0731 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --tensor-parallel-size 4 \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 2048 \\\n --gpu-memory-utilization 0.7 \\\n --enable-prefix-caching \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --reasoning-config '{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}' \\\n --default-chat-template-kwargs.thinking=true \\\n --default-chat-template-kwargs.reasoning_effort=high \\\n --load-format b12x \\\n --moe-backend b12x \\\n --linear-backend b12x \\\n --attention-backend B12X \\\n --max-cudagraph-capture-size 48 \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}' \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":5,\"draft_sample_method\":\"probabilistic\",\"attention_backend\":\"B12X\",\"draft_tensor_parallel_size\":4}' \\\n --kv-cache-memory-bytes 4G \\\n --nnodes 4 \\\n --node-rank 0 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\"", "port": 8000, "api_model": "deepseek-ai/DeepSeek-V4-Flash-0731", "additional_commands": [ { "id": "workers", "label": "Launch workers", "target": "workers", "instructions": "Run inside each worker container with a unique NODE_RANK of 1, 2, or 3. Trailing flags preserve the source runner's overrides; the last value of each repeated flag takes effect.", "command": "# Use a different rank (1, 2, or 3) on each worker.\nexport NODE_RANK=\"\"\n\nexport CUTE_DSL_ARCH=\"sm_121a\"\nexport VLLM_USE_AOT_COMPILE=\"1\"\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=\"0\"\nexport VLLM_USE_MEGA_AOT_ARTIFACT=\"1\"\nexport VLLM_MEMORY_PROFILE_INCLUDE_ATTN=\"1\"\nexport VLLM_USE_FLASHINFER_SAMPLER=\"1\"\nexport VLLM_USE_B12X_WO_PROJECTION=\"1\"\nexport VLLM_USE_B12X_MHC=\"1\"\nexport VLLM_USE_B12X_FP8_GEMM=\"1\"\nexport VLLM_USE_B12X_MOE=\"1\"\nexport VLLM_USE_B12X_SPARSE_INDEXER=\"1\"\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\nexport VLLM_MOE_SKIP_PADDING=\"0\"\nexport B12X_MLA_SM120_UNIFIED=\"1\"\nexport B12X_MOE_FORCE_A8=\"1\"\n\nvllm serve deepseek-ai/DeepSeek-V4-Flash-0731 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --tensor-parallel-size 4 \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 2048 \\\n --gpu-memory-utilization 0.7 \\\n --enable-prefix-caching \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --reasoning-config '{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}' \\\n --default-chat-template-kwargs.thinking=true \\\n --default-chat-template-kwargs.reasoning_effort=high \\\n --load-format b12x \\\n --moe-backend b12x \\\n --linear-backend b12x \\\n --attention-backend B12X \\\n --max-cudagraph-capture-size 48 \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}' \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":5,\"draft_sample_method\":\"probabilistic\",\"attention_backend\":\"B12X\",\"draft_tensor_parallel_size\":4}' \\\n --kv-cache-memory-bytes 4G \\\n --nnodes 4 \\\n --node-rank \"$NODE_RANK\" \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\" \\\n --headless" } ] }, "launch": null, "sampling": null, "notes": "Use the same CX7 port on every node; assign both port subinterfaces on separate subnets; cache the image and checkpoint on every node; start all three workers before the head. Docker runs privileged.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"deepseek-ai/DeepSeek-V4-Flash-0731\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"deepseek-ai/DeepSeek-V4-Flash-0731\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "deepseek/deepseek-v4-flash-0731/dgx-station/linux/vllm/fp4-fp8-mixed", "hardware": "dgx-station", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "v0.31.0", "backend_min_version": "v0.25.1", "min_vidmem_gb": null, "precision": "fp4-fp8-mixed", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731", "target_label": "deepseek-ai/DeepSeek-V4-Flash-0731", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Station", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.31.0", "image_url": "https://hub.docker.com/layers/vllm/vllm-openai/v0.31.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set HF_TOKEN in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.31.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve deepseek-ai/DeepSeek-V4-Flash-0731 \\\n --served-model-name deepseek-v4-flash-0731 \\\n --host 0.0.0.0 --port 8000 \\\n --trust-remote-code \\\n --tensor-parallel-size 1 \\\n --kv-cache-dtype fp8_ds_mla \\\n --block-size 256 \\\n --max-model-len 1048576 \\\n --max-num-seqs 64 \\\n --max-num-batched-tokens 32768 \\\n --max-cudagraph-capture-size 512 \\\n --gpu-memory-utilization 0.88 \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --async-scheduling \\\n --enable-chunked-prefill \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}' \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --generation-config vllm \\\n --compilation-config '{\"custom_ops\":[\"all\"]}'", "port": 8000, "api_model": "deepseek-v4-flash-0731" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"deepseek-v4-flash-0731\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"deepseek-v4-flash-0731\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "deepseek/deepseek-v4-flash-0731/rtx-pro-2gpu/linux/vllm/fp4-fp8-mixed", "hardware": "rtx-pro-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21830+g8244d1c44.cu134", "backend_min_version": null, "min_vidmem_gb": 96, "precision": "fp4-fp8-mixed", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731", "target_label": "deepseek-ai/DeepSeek-V4-Flash-0731", "draft": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731", "draft_label": "deepseek-ai/DeepSeek-V4-Flash-0731" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× RTX Pro 6000", "gpu_count": 2, "node_count": 1, "tensor_parallel_size": 2 }, "runtime": { "kind": "container", "label": "Container (community)", "image": "ghcr.io/local-inference-lab/vllm:release-10152026", "image_url": "https://github.com/local-inference-lab/blackwell-llm-docker/pkgs/container/vllm/1392689728?tag=release-10152026", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set HF_TOKEN in the host environment", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n ghcr.io/local-inference-lab/vllm:release-10152026" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "export CUDA_DEVICE_ORDER=PCI_BUS_ID\nexport CUTE_DSL_ARCH=sm_120a\nexport HF_HOME=/root/.cache/huggingface\nexport INSTANTTENSOR_BACKEND=URING,AIO\nexport NCCL_BUFFSIZE=2097152\nexport NCCL_IB_DISABLE=1\nexport NCCL_MAX_NCHANNELS=16\nexport NCCL_MIN_NCHANNELS=16\nexport NCCL_NET_PLUGIN=spcx\nexport NCCL_P2P_LEVEL=SYS\nexport NCCL_PROTO=LL,LL128,Simple\nexport OMP_NUM_THREADS=2\nexport PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport SAFETENSORS_FAST_GPU=1\nexport VLLM_B12X_MOE_FP4_FORCE_A16=0\nexport VLLM_ENABLE_PCIE_ALLREDUCE=1\nexport VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1\nexport VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD=1024\nexport VLLM_PCIE_ALLREDUCE_BACKEND=b12x\nexport VLLM_PCIE_ONESHOT_ALLREDUCE_MAX_SIZE=64KB\nexport VLLM_USE_AOT_COMPILE=1\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=0\nexport VLLM_USE_FLASHINFER_SAMPLER=1\nexport VLLM_USE_MEGA_AOT_ARTIFACT=1\nexport VLLM_USE_V2_MODEL_RUNNER=1\nexport VLLM_WORKER_MULTIPROC_METHOD=spawn\nvllm serve deepseek-ai/DeepSeek-V4-Flash-0731 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --async-scheduling \\\n --attention-backend B12X \\\n --block-size 256 \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}' \\\n --tensor-parallel-size 2 \\\n --decode-context-parallel-size 1 \\\n --default-chat-template-kwargs '{\"thinking\":true,\"reasoning_effort\":\"high\"}' \\\n --dtype bfloat16 \\\n --enable-auto-tool-choice \\\n --enable-chunked-prefill \\\n --enable-flashinfer-autotune \\\n --enable-force-include-usage \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-request-id-headers \\\n --gpu-memory-utilization 0.975 \\\n --kv-cache-dtype fp8 \\\n --load-format instanttensor \\\n --max-cudagraph-capture-size 48 \\\n --max-model-len auto \\\n --max-num-batched-tokens 4096 \\\n --max-num-seqs 8 \\\n --moe-backend b12x \\\n --override-generation-config '{\"temperature\":1.0,\"top_p\":0.95}' \\\n --pipeline-parallel-size 1 \\\n --prefix-cache-retention-interval 4096 \\\n --reasoning-parser deepseek_v4 \\\n --no-scheduler-reserve-full-isl \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --trust-remote-code \\\n --speculative-config '{\"method\":\"dspark\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"standard\",\"num_speculative_tokens\":5,\"model\":\"deepseek-ai/DeepSeek-V4-Flash-0731\"}'", "port": 8000, "api_model": "deepseek-ai/DeepSeek-V4-Flash-0731" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"deepseek-ai/DeepSeek-V4-Flash-0731\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"deepseek-ai/DeepSeek-V4-Flash-0731\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "deepseek/deepseek-v4-flash-vision-exp", "display_name": "DeepSeek-V4-Flash-Vision-Exp", "builder": "deepseek", "family": "DeepSeek-V4", "summary": "DeepSeek V4 Flash experimental multimodal model with vision support.", "upstream": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "modalities": [ "image", "text" ], "recipes": [ { "id": "deepseek/deepseek-v4-flash-vision-exp/dgx-spark-2gpu/linux/vllm/fp4-fp8-mixed", "hardware": "dgx-spark-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21830+g8244d1c44.d20261015", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "fp4-fp8-mixed", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [ "The source requires the custom B12x container image." ], "checkpoint": { "target": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "target_label": "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× DGX Spark; one node rank per Spark; tensor parallel 2", "gpu_count": 2, "node_count": 2, "tensor_parallel_size": 2 }, "runtime": { "kind": "container", "label": "Container (community, mutable, ARM64, custom B12x)", "image": "eugr/spark-vllm-b12x:latest", "image_url": "https://hub.docker.com/r/eugr/spark-vllm-b12x/tags?name=latest", "setup_label": "Start container", "setup_target": "all-nodes", "setup_instructions": "Run on every node after setting `HF_TOKEN`; replace the node-specific placeholders.", "setup_command": "# Example two-Spark values (adjust to your network):\n# HEAD_IP=\"10.100.32.1\" # head node IP on the ETH subnet\n# ETH_IF=\"enp1s0f0np0\" # ConnectX baseboard ethernet (socket/control)\n# IB_IF=\"rocep1s0f0,roceP2p1s0f0\" # RoCE HCA names (data plane) - use both \"twins\"!\nexport HEAD_IP=\"\"\nexport ETH_IF=\"\"\nexport IB_IF=\",\"\nexport MASTER_PORT=29501\nexport IMAGE=\"eugr/spark-vllm-b12x:latest\"\nexport HF_CACHE_DIR=\"$HOME/.cache/huggingface\"\n\nmkdir -p \"$HF_CACHE_DIR\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\" \"$HOME/.cache/b12x\"\n\ndocker run --rm -it \\\n --privileged \\\n --security-opt seccomp=unconfined \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit nofile=1048576:1048576 \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HF_CACHE_DIR:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n -v $HOME/.cache/b12x:/root/.cache/b12x \\\n --entrypoint /bin/bash \\\n \"$IMAGE\"" }, "serve": { "label": "Launch head", "target": "head", "instructions": null, "command": "export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport CUTE_DSL_ARCH=\"sm_121a\"\nexport VLLM_USE_AOT_COMPILE=\"1\"\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=\"0\"\nexport VLLM_USE_MEGA_AOT_ARTIFACT=\"1\"\nexport VLLM_MEMORY_PROFILE_INCLUDE_ATTN=\"1\"\nexport VLLM_USE_FLASHINFER_SAMPLER=\"1\"\nexport VLLM_USE_B12X_WO_PROJECTION=\"1\"\nexport VLLM_USE_B12X_MHC=\"1\"\nexport VLLM_USE_B12X_FP8_GEMM=\"1\"\nexport VLLM_USE_B12X_MOE=\"1\"\nexport VLLM_USE_B12X_SPARSE_INDEXER=\"1\"\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\nexport B12X_MLA_SM120_UNIFIED=\"1\"\nexport B12X_MOE_FORCE_A8=\"1\"\nexport VLLM_MOE_SKIP_PADDING=\"0\"\nexport VLLM_ENABLE_ROCE_ALLREDUCE=1\nexport VLLM_ROCE_ALLREDUCE_MAX_SIZE=2MB\nvllm serve deepseek-ai/DeepSeek-V4-Flash-Vision-Exp \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --tensor-parallel-size 2 \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --max-model-len auto \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --gpu-memory-utilization 0.85 \\\n --enable-prefix-caching \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --reasoning-config '{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}' \\\n --default-chat-template-kwargs.thinking=true \\\n --default-chat-template-kwargs.reasoning_effort=high \\\n --load-format b12x \\\n --moe-backend b12x \\\n --linear-backend b12x \\\n --attention-backend B12X \\\n --max-cudagraph-capture-size 48 \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}' \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\",\"attention_backend\":\"B12X\"}' \\\n --nnodes 2 \\\n --node-rank 0 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\"", "port": 8000, "api_model": "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "additional_commands": [ { "id": "worker", "label": "Launch worker", "target": "workers", "instructions": null, "command": "export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport CUTE_DSL_ARCH=\"sm_121a\"\nexport VLLM_USE_AOT_COMPILE=\"1\"\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=\"0\"\nexport VLLM_USE_MEGA_AOT_ARTIFACT=\"1\"\nexport VLLM_MEMORY_PROFILE_INCLUDE_ATTN=\"1\"\nexport VLLM_USE_FLASHINFER_SAMPLER=\"1\"\nexport VLLM_USE_B12X_WO_PROJECTION=\"1\"\nexport VLLM_USE_B12X_MHC=\"1\"\nexport VLLM_USE_B12X_FP8_GEMM=\"1\"\nexport VLLM_USE_B12X_MOE=\"1\"\nexport VLLM_USE_B12X_SPARSE_INDEXER=\"1\"\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\nexport B12X_MLA_SM120_UNIFIED=\"1\"\nexport B12X_MOE_FORCE_A8=\"1\"\nexport VLLM_MOE_SKIP_PADDING=\"0\"\nexport VLLM_ENABLE_ROCE_ALLREDUCE=1\nexport VLLM_ROCE_ALLREDUCE_MAX_SIZE=2MB\nvllm serve deepseek-ai/DeepSeek-V4-Flash-Vision-Exp \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --tensor-parallel-size 2 \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --max-model-len auto \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --gpu-memory-utilization 0.85 \\\n --enable-prefix-caching \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --reasoning-config '{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}' \\\n --default-chat-template-kwargs.thinking=true \\\n --default-chat-template-kwargs.reasoning_effort=high \\\n --load-format b12x \\\n --moe-backend b12x \\\n --linear-backend b12x \\\n --attention-backend B12X \\\n --max-cudagraph-capture-size 48 \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}' \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\",\"attention_backend\":\"B12X\"}' \\\n --nnodes 2 \\\n --node-rank 1 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\" \\\n --headless" } ] }, "launch": null, "sampling": null, "notes": "Use the same CX7 port on every node; assign both port subinterfaces on separate subnets; cache the image and checkpoint on every node; start the worker before the head. Docker runs privileged.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "deepseek/deepseek-v4-flash-vision-exp/dgx-spark-4gpu/linux/vllm/64gb/fp4-fp8-mixed", "hardware": "dgx-spark-4gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21830+g8244d1c44.d20261015", "backend_min_version": null, "min_vidmem_gb": 64, "precision": "fp4-fp8-mixed", "variant": null, "status": "optimized", "prerequisites": [ "Four DGX Spark nodes with at least 64 GB of memory per node and a configured RoCE network.", "Cache the complete checkpoint, tokenizer, and processor files on every node before launch; HF_HUB_OFFLINE=1 and TRANSFORMERS_OFFLINE=1 disable downloads." ], "limitations": [ "The source requires the custom B12x container image." ], "checkpoint": { "target": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "target_label": "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "4× DGX Spark; 64 GB per node; one node rank per Spark; tensor parallel 4", "gpu_count": 4, "node_count": 4, "tensor_parallel_size": 4 }, "runtime": { "kind": "container", "label": "Container (community, mutable, ARM64, custom B12x)", "image": "eugr/spark-vllm-b12x:latest", "image_url": "https://hub.docker.com/r/eugr/spark-vllm-b12x/tags?name=latest", "setup_label": "Start container", "setup_target": "all-nodes", "setup_instructions": "Run on every node after setting `HF_TOKEN`; replace the node-specific placeholders.", "setup_command": "# Example four-Spark values (adjust to your network):\n# HEAD_IP=\"10.100.32.1\" # head node IP on the ETH subnet\n# ETH_IF=\"enp1s0f0np0\" # ConnectX baseboard ethernet (socket/control)\n# IB_IF=\"rocep1s0f0,roceP2p1s0f0\" # RoCE HCA names (data plane) - use both \"twins\"!\nexport HEAD_IP=\"\"\nexport ETH_IF=\"\"\nexport IB_IF=\",\"\nexport MASTER_PORT=29501\nexport IMAGE=\"eugr/spark-vllm-b12x:latest\"\nexport HF_CACHE_DIR=\"$HOME/.cache/huggingface\"\n\nmkdir -p \"$HF_CACHE_DIR\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\" \"$HOME/.cache/b12x\"\n\ndocker run --rm -it \\\n --privileged \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit nofile=1048576:1048576 \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \\\n -e NCCL_BUFFSIZE=1048576 \\\n -e NCCL_MAX_CTAS=8 \\\n -e VLLM_ENABLE_ROCE_ALLREDUCE=1 \\\n -e VLLM_ROCE_ALLREDUCE_MAX_SIZE=2MB \\\n -e MIMALLOC_PURGE_DELAY=0 \\\n -e B12X_COMPILE_WORKERS=1 \\\n -e VLLM_USE_RUST_FRONTEND=0 \\\n -e HF_HUB_OFFLINE=1 \\\n -e TRANSFORMERS_OFFLINE=1 \\\n -e MAX_JOBS=1 \\\n -e CMAKE_BUILD_PARALLEL_LEVEL=1 \\\n -e NINJAFLAGS=-j1 \\\n -e MAKEFLAGS=-j1 \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HF_CACHE_DIR:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n -v $HOME/.cache/b12x:/root/.cache/b12x \\\n --entrypoint /bin/bash \\\n \"$IMAGE\"" }, "serve": { "label": "Launch head", "target": "head", "instructions": "Run inside the head container after starting all three workers.", "command": "export CUTE_DSL_ARCH=\"sm_121a\"\nexport VLLM_USE_AOT_COMPILE=\"1\"\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=\"0\"\nexport VLLM_USE_MEGA_AOT_ARTIFACT=\"1\"\nexport VLLM_MEMORY_PROFILE_INCLUDE_ATTN=\"1\"\nexport VLLM_USE_FLASHINFER_SAMPLER=\"1\"\nexport VLLM_USE_B12X_WO_PROJECTION=\"1\"\nexport VLLM_USE_B12X_MHC=\"1\"\nexport VLLM_USE_B12X_FP8_GEMM=\"1\"\nexport VLLM_USE_B12X_MOE=\"1\"\nexport VLLM_USE_B12X_SPARSE_INDEXER=\"1\"\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\nexport VLLM_MOE_SKIP_PADDING=\"0\"\nexport B12X_MLA_SM120_UNIFIED=\"1\"\nexport B12X_MOE_FORCE_A8=\"1\"\n\nvllm serve deepseek-ai/DeepSeek-V4-Flash-Vision-Exp \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --tensor-parallel-size 4 \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 2048 \\\n --gpu-memory-utilization 0.7 \\\n --enable-prefix-caching \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --reasoning-config '{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}' \\\n --default-chat-template-kwargs.thinking=true \\\n --default-chat-template-kwargs.reasoning_effort=high \\\n --load-format b12x \\\n --moe-backend b12x \\\n --linear-backend b12x \\\n --attention-backend B12X \\\n --max-cudagraph-capture-size 48 \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}' \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":5,\"draft_sample_method\":\"probabilistic\",\"attention_backend\":\"B12X\",\"draft_tensor_parallel_size\":4}' \\\n --kv-cache-memory-bytes 3G \\\n --limit-mm-per-prompt '{\"image\":2}' \\\n --mm-processor-cache-gb 0 \\\n --mm-encoder-tp-mode weights \\\n --nnodes 4 \\\n --node-rank 0 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\"", "port": 8000, "api_model": "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "additional_commands": [ { "id": "workers", "label": "Launch workers", "target": "workers", "instructions": "Run inside each worker container with a unique NODE_RANK of 1, 2, or 3. Trailing flags preserve the source runner's overrides; the last value of each repeated flag takes effect.", "command": "# Use a different rank (1, 2, or 3) on each worker.\nexport NODE_RANK=\"\"\n\nexport CUTE_DSL_ARCH=\"sm_121a\"\nexport VLLM_USE_AOT_COMPILE=\"1\"\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=\"0\"\nexport VLLM_USE_MEGA_AOT_ARTIFACT=\"1\"\nexport VLLM_MEMORY_PROFILE_INCLUDE_ATTN=\"1\"\nexport VLLM_USE_FLASHINFER_SAMPLER=\"1\"\nexport VLLM_USE_B12X_WO_PROJECTION=\"1\"\nexport VLLM_USE_B12X_MHC=\"1\"\nexport VLLM_USE_B12X_FP8_GEMM=\"1\"\nexport VLLM_USE_B12X_MOE=\"1\"\nexport VLLM_USE_B12X_SPARSE_INDEXER=\"1\"\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\nexport VLLM_MOE_SKIP_PADDING=\"0\"\nexport B12X_MLA_SM120_UNIFIED=\"1\"\nexport B12X_MOE_FORCE_A8=\"1\"\n\nvllm serve deepseek-ai/DeepSeek-V4-Flash-Vision-Exp \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --tensor-parallel-size 4 \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 2048 \\\n --gpu-memory-utilization 0.7 \\\n --enable-prefix-caching \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --reasoning-config '{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}' \\\n --default-chat-template-kwargs.thinking=true \\\n --default-chat-template-kwargs.reasoning_effort=high \\\n --load-format b12x \\\n --moe-backend b12x \\\n --linear-backend b12x \\\n --attention-backend B12X \\\n --max-cudagraph-capture-size 48 \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}' \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":5,\"draft_sample_method\":\"probabilistic\",\"attention_backend\":\"B12X\",\"draft_tensor_parallel_size\":4}' \\\n --kv-cache-memory-bytes 3G \\\n --limit-mm-per-prompt '{\"image\":2}' \\\n --mm-processor-cache-gb 0 \\\n --mm-encoder-tp-mode weights \\\n --nnodes 4 \\\n --node-rank \"$NODE_RANK\" \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\" \\\n --headless" } ] }, "launch": null, "sampling": null, "notes": "Use the same CX7 port on every node; assign both port subinterfaces on separate subnets; cache the image and checkpoint on every node; start all three workers before the head. Docker runs privileged.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "deepseek/deepseek-v4-flash-vision-exp/dgx-station/linux/vllm/fp4-fp8-mixed", "hardware": "dgx-station", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.28.1rc1.dev137+g5ab628dd1", "backend_min_version": null, "min_vidmem_gb": null, "precision": "fp4-fp8-mixed", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "target_label": "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "Single DGX Station", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:deepseekv4-flash-vision", "image_url": "https://hub.docker.com/layers/vllm/vllm-openai/deepseekv4-flash-vision", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Download the model from Hugging Face on the host machine before running the container.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:deepseekv4-flash-vision" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve deepseek-ai/DeepSeek-V4-Flash-Vision-Exp \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --tensor-parallel-size 1 \\\n --kv-cache-dtype fp8_ds_mla \\\n --block-size 256 \\\n --max-model-len 1048576 \\\n --max-num-seqs 16 \\\n --max-num-batched-tokens 32768 \\\n --max-cudagraph-capture-size 512 \\\n --gpu-memory-utilization 0.92 \\\n --enable-prefix-caching \\\n --async-scheduling \\\n --enable-chunked-prefill \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\",\"enable_adaptive_verification\":true}' \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --reasoning-config '{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}' \\\n --generation-config vllm \\\n --compilation-config '{\"custom_ops\":[\"all\"]}'", "port": 8000, "api_model": "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "deepseek/deepseek-v4-flash-vision-exp/rtx-pro-2gpu/linux/vllm/fp4-fp8-mixed", "hardware": "rtx-pro-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21830+g8244d1c44.cu134", "backend_min_version": null, "min_vidmem_gb": 96, "precision": "fp4-fp8-mixed", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "target_label": "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "draft": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "draft_label": "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× RTX Pro 6000", "gpu_count": 2, "node_count": 1, "tensor_parallel_size": 2 }, "runtime": { "kind": "container", "label": "Container (community)", "image": "ghcr.io/local-inference-lab/vllm:release-10152026", "image_url": "https://github.com/local-inference-lab/blackwell-llm-docker/pkgs/container/vllm/1392689728?tag=release-10152026", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set HF_TOKEN in the host environment", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n ghcr.io/local-inference-lab/vllm:release-10152026" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "export CUDA_VISIBLE_DEVICES=0,1\nexport CUDA_DEVICE_ORDER=PCI_BUS_ID\nexport CUDA_MODULE_LOADING=LAZY\nexport CUTE_DSL_ARCH=sm_120a\nexport OMP_NUM_THREADS=2\nexport NCCL_IB_DISABLE=1\nexport NCCL_P2P_LEVEL=SYS\nexport NCCL_PROTO=LL,LL128,Simple\nexport NCCL_BUFFSIZE=2097152\nexport NCCL_MIN_NCHANNELS=16\nexport NCCL_MAX_NCHANNELS=16\nexport INSTANTTENSOR_BACKEND=BUFFERED\nexport SAFETENSORS_FAST_GPU=1\nexport PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1\nexport TORCH_NCCL_USE_COMM_NONBLOCKING=0\nexport VLLM_WORKER_MULTIPROC_METHOD=spawn\nexport VLLM_USE_AOT_COMPILE=1\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=0\nexport VLLM_USE_MEGA_AOT_ARTIFACT=1\nexport VLLM_USE_FLASHINFER_SAMPLER=1\nexport VLLM_USE_V2_MODEL_RUNNER=1\nexport VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1\nexport VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD=1024\nexport VLLM_ENABLE_PCIE_ALLREDUCE=1\nexport VLLM_PCIE_ALLREDUCE_BACKEND=b12x\nexport VLLM_PCIE_ONESHOT_ALLREDUCE_MAX_SIZE=64KB\nexport VLLM_B12X_MOE_FP4_FORCE_A16=0\n\nvllm serve deepseek-ai/DeepSeek-V4-Flash-Vision-Exp \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --dtype bfloat16 \\\n --load-format instanttensor \\\n --tensor-parallel-size 2 \\\n --pipeline-parallel-size 1 \\\n --decode-context-parallel-size 1 \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --gpu-memory-utilization 0.975 \\\n --max-model-len auto \\\n --max-num-seqs 4 \\\n --max-num-batched-tokens 4096 \\\n --max-cudagraph-capture-size 16 \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}' \\\n --attention-backend B12X \\\n --moe-backend b12x \\\n --enable-flashinfer-autotune \\\n --async-scheduling \\\n --no-scheduler-reserve-full-isl \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --prefix-cache-retention-interval 4096 \\\n --tokenizer-mode deepseek_v4 \\\n --reasoning-parser deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --default-chat-template-kwargs '{\"thinking\":true,\"reasoning_effort\":\"high\"}' \\\n --override-generation-config '{\"temperature\":1.0,\"top_p\":0.95}' \\\n --enable-prompt-tokens-details \\\n --enable-force-include-usage \\\n --enable-request-id-headers \\\n --speculative-config '{\"method\":\"dspark\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"standard\",\"num_speculative_tokens\":3,\"model\":\"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp\"}'", "port": 8000, "api_model": "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "google/diffusiongemma-26b-a4b", "display_name": "DiffusionGemma-26B-A4B", "builder": "google", "family": "DiffusionGemma", "summary": "Google DiffusionGemma (26B) model.", "upstream": null, "modalities": [ "text" ], "recipes": [ { "id": "google/diffusiongemma-26b-a4b/dgx-spark/linux/vllm/nvfp4", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21818+g3a947437c.d20261014", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "nvfp4", "variant": null, "status": "optimized", "prerequisites": [ "Docker with NVIDIA GPU access and Hugging Face access to the checkpoint." ], "limitations": [ "Published coding performance was measured at concurrency 1; AgentPerf was not included." ], "checkpoint": { "target": "https://huggingface.co/nvidia/diffusiongemma-26B-A4B-it-NVFP4", "target_label": "nvidia/diffusiongemma-26B-A4B-it-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container (community, mutable, ARM64)", "image": "eugr/spark-vllm:nightly-20261014", "image_url": "https://hub.docker.com/r/eugr/spark-vllm/tags?name=nightly-20261014", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n eugr/spark-vllm:nightly-20261014" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/diffusiongemma-26B-A4B-it-NVFP4 \\\n --max-model-len 262144 \\\n --gpu-memory-utilization 0.8 \\\n --port 8000 \\\n --host 0.0.0.0 \\\n --trust-remote-code \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --load-format fastsafetensors \\\n --attention-backend TRITON_ATTN \\\n --max-num-seqs 10 \\\n --enable-prefix-caching \\\n --diffusion-config '{\"canvas_length\":256}' \\\n --override-generation-config '{\"max_new_tokens\": null}' \\\n --reasoning-parser gemma4 \\\n --default-chat-template-kwargs '{\"enable_thinking\": true}'", "port": 8000, "api_model": "nvidia/diffusiongemma-26B-A4B-it-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/diffusiongemma-26B-A4B-it-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/diffusiongemma-26B-A4B-it-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "google/gemma-4-26b-a4b", "display_name": "Gemma-4-26B-A4B", "builder": "google", "family": "Gemma 4", "summary": "Google Gemma 4 (26B-A4B) text-generation model.", "upstream": null, "modalities": [ "text", "image" ], "recipes": [ { "id": "google/gemma-4-26b-a4b/dgx-spark/linux/vllm/128gb/nvfp4/mtp", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.32.0rc1.dev301+g1ac942555.d20261016", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "nvfp4", "variant": "mtp", "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Gemma-4-26B-A4B-NVFP4", "target_label": "nvidia/Gemma-4-26B-A4B-NVFP4", "draft": "https://huggingface.co/google/gemma-4-26B-A4B-it-assistant", "draft_label": "google/gemma-4-26B-A4B-it-assistant" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container (community, ARM64)", "image": "eugr/spark-vllm:nightly-20261016", "image_url": "https://hub.docker.com/layers/eugr/spark-vllm/nightly-20261016", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n eugr/spark-vllm:nightly-20261016" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Gemma-4-26B-A4B-NVFP4 \\\n --max-model-len 262144 \\\n --gpu-memory-utilization 0.8 \\\n --port 8000 \\\n --host 0.0.0.0 \\\n --max-num-seqs 8 \\\n --load-format instanttensor \\\n --enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --reasoning-parser gemma4 \\\n --max-num-batched-tokens 4096 \\\n --speculative-config '{\"method\":\"mtp\",\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4, \"moe_backend\": \"triton\"}'", "port": 8000, "api_model": "nvidia/Gemma-4-26B-A4B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Gemma-4-26B-A4B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Gemma-4-26B-A4B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "google/gemma-4-26b-a4b/dgx-spark/linux/vllm/64gb/nvfp4/mtp", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.32.0rc1.dev301+g1ac942555.d20261016", "backend_min_version": null, "min_vidmem_gb": 64, "precision": "nvfp4", "variant": "mtp", "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Gemma-4-26B-A4B-NVFP4", "target_label": "nvidia/Gemma-4-26B-A4B-NVFP4", "draft": "https://huggingface.co/google/gemma-4-26B-A4B-it-assistant", "draft_label": "google/gemma-4-26B-A4B-it-assistant" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container (community, ARM64)", "image": "eugr/spark-vllm:nightly-20261016", "image_url": "https://hub.docker.com/layers/eugr/spark-vllm/nightly-20261016", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n eugr/spark-vllm:nightly-20261016" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Gemma-4-26B-A4B-NVFP4 \\\n --max-model-len 262144 \\\n --gpu-memory-utilization 0.8 \\\n --port 8000 \\\n --host 0.0.0.0 \\\n --max-num-seqs 8 \\\n --load-format instanttensor \\\n --enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --reasoning-parser gemma4 \\\n --max-num-batched-tokens 4096 \\\n --speculative-config '{\"method\":\"mtp\",\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4, \"moe_backend\": \"triton\"}'", "port": 8000, "api_model": "nvidia/Gemma-4-26B-A4B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Gemma-4-26B-A4B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Gemma-4-26B-A4B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "google/gemma-4-26b-a4b/dgx-station/linux/vllm/nvfp4/mtp", "hardware": "dgx-station", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.30.0", "backend_min_version": null, "min_vidmem_gb": 250, "precision": "nvfp4", "variant": "mtp", "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Gemma-4-26B-A4B-NVFP4", "target_label": "nvidia/Gemma-4-26B-A4B-NVFP4", "draft": "https://huggingface.co/google/gemma-4-26B-A4B-it-assistant", "draft_label": "google/gemma-4-26B-A4B-it-assistant" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Station", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.30.0", "image_url": "https://hub.docker.com/layers/vllm/vllm-openai/v0.30.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.30.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Gemma-4-26B-A4B-NVFP4 \\\n --max-model-len 262144 \\\n --gpu-memory-utilization 0.92 \\\n --port 8000 \\\n --host 0.0.0.0 \\\n --max-num-seqs 64 \\\n --load-format instanttensor \\\n --enable-prefix-caching \\\n --enable-chunked-prefill \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --reasoning-parser gemma4 \\\n --max-num-batched-tokens 16384 \\\n --speculative-config '{\"method\":\"mtp\",\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4, \"moe_backend\": \"triton\"}'", "port": 8000, "api_model": "nvidia/Gemma-4-26B-A4B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Gemma-4-26B-A4B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Gemma-4-26B-A4B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "google/gemma-4-26b-a4b/geforce-rtx/linux/llama-cpp/q4-k-m/mtp", "hardware": "geforce-rtx", "platform": { "os": "linux", "execution": "native" }, "backend": "llama-cpp", "backend_version": null, "backend_min_version": "b10505", "min_vidmem_gb": 20, "precision": "q4-k-m", "variant": "mtp", "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "target_label": "unsloth/gemma-4-26B-A4B-it-GGUF", "draft": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/mtp-gemma-4-26B-A4B-it.gguf", "draft_label": "unsloth/gemma-4-26B-A4B-it-GGUF/mtp-gemma-4-26B-A4B-it.gguf" }, "security": null, "topology": null, "runtime": null, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in a Linux shell. llama-server downloads and caches the target and MTP draft GGUFs from Hugging Face.", "command": "llama-server -hf \"unsloth/gemma-4-26B-A4B-it-GGUF\" \\\n -hff \"gemma-4-26B-A4B-it-UD-Q4_K_M.gguf\" \\\n --no-mmproj \\\n -hfrd \"unsloth/gemma-4-26B-A4B-it-GGUF\" \\\n --backend-sampling \\\n --batch-size 2048 \\\n --ubatch-size 2048 \\\n --ctx-size 65536 \\\n --parallel 1 \\\n --spec-type draft-mtp \\\n --spec-draft-n-max 3 \\\n --spec-draft-model \"mtp-gemma-4-26B-A4B-it.gguf\" \\\n --temp 1.0 \\\n --top-p 0.95 \\\n --top-k 64", "port": null, "api_model": "unsloth/gemma-4-26B-A4B-it-GGUF" }, "launch": null, "sampling": "Target backend sampling: ON; speculative draft backend sampling: default; --temp=1.0; --top-p=0.95; --top-k=64", "notes": "The public [target GGUF](https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf) uses the public [MTP draft GGUF](https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/mtp-gemma-4-26B-A4B-it.gguf). Use MTP depth 3.", "clients": null }, { "id": "google/gemma-4-26b-a4b/geforce-rtx/linux/vllm/nvfp4/mtp", "hardware": "geforce-rtx", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": null, "backend_min_version": null, "min_vidmem_gb": 32, "precision": "nvfp4", "variant": "mtp", "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Gemma-4-26B-A4B-NVFP4", "target_label": "nvidia/Gemma-4-26B-A4B-NVFP4", "draft": "https://huggingface.co/google/gemma-4-26B-A4B-it-assistant", "draft_label": "google/gemma-4-26B-A4B-it-assistant" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× GeForce RTX GPUs", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.30.0", "image_url": "https://hub.docker.com/layers/vllm/vllm-openai/v0.30.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.30.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Gemma-4-26B-A4B-NVFP4 \\\n --max-model-len 262144 \\\n --gpu-memory-utilization 0.92 \\\n --port 8000 \\\n --host 0.0.0.0 \\\n --max-num-seqs 1 \\\n --enable-prefix-caching \\\n --tool-call-parser gemma4 \\\n --reasoning-parser gemma4 \\\n --enable-auto-tool-choice \\\n --max-num-batched-tokens 4096 \\\n --speculative-config '{\"method\":\"mtp\",\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4, \"moe_backend\": \"triton\"}'", "port": 8000, "api_model": "nvidia/Gemma-4-26B-A4B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Gemma-4-26B-A4B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Gemma-4-26B-A4B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "google/gemma-4-26b-a4b/geforce-rtx/windows/llama-cpp/24gb/q4-k-m/vision-mtp", "hardware": "geforce-rtx", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11256", "min_vidmem_gb": 24, "precision": "q4-k-m", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoints and projector." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "target_label": "unsloth/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "draft": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/mtp-gemma-4-26B-A4B-it.gguf", "draft_label": "unsloth/gemma-4-26B-A4B-it-GGUF/mtp-gemma-4-26B-A4B-it.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target, draft, and projector GGUFs from Hugging Face.", "command": "llama-server.exe -hf \"unsloth/gemma-4-26B-A4B-it-GGUF\" `\n -hff \"gemma-4-26B-A4B-it-UD-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/resolve/c099eb48e663fd284577b04978a94ffccb261841/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 1024 `\n --ctx-size 172032 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 3 `\n -hfrd \"unsloth/gemma-4-26B-A4B-it-GGUF\" `\n --spec-draft-model \"mtp-gemma-4-26B-A4B-it.gguf\" `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 64", "port": 9931, "api_model": "unsloth/gemma-4-26B-A4B-it-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=64", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/c099eb48e663fd284577b04978a94ffccb261841/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/gemma-4-26B-A4B-it-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/gemma-4-26B-A4B-it-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "google/gemma-4-26b-a4b/geforce-rtx/windows/llama-cpp/26gb/q4-k-m/vision-mtp", "hardware": "geforce-rtx", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11256", "min_vidmem_gb": 26, "precision": "q4-k-m", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoints and projector." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "target_label": "unsloth/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "draft": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/mtp-gemma-4-26B-A4B-it.gguf", "draft_label": "unsloth/gemma-4-26B-A4B-it-GGUF/mtp-gemma-4-26B-A4B-it.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target, draft, and projector GGUFs from Hugging Face.", "command": "llama-server.exe -hf \"unsloth/gemma-4-26B-A4B-it-GGUF\" `\n -hff \"gemma-4-26B-A4B-it-UD-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/resolve/c099eb48e663fd284577b04978a94ffccb261841/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 1024 `\n --ctx-size 262144 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 3 `\n -hfrd \"unsloth/gemma-4-26B-A4B-it-GGUF\" `\n --spec-draft-model \"mtp-gemma-4-26B-A4B-it.gguf\" `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 64", "port": 9931, "api_model": "unsloth/gemma-4-26B-A4B-it-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=64", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/c099eb48e663fd284577b04978a94ffccb261841/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/gemma-4-26B-A4B-it-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/gemma-4-26B-A4B-it-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "google/gemma-4-26b-a4b/rtx-pro/linux/llama-cpp/q4-k-m/mtp", "hardware": "rtx-pro", "platform": { "os": "linux", "execution": "native" }, "backend": "llama-cpp", "backend_version": null, "backend_min_version": "b10505", "min_vidmem_gb": 20, "precision": "q4-k-m", "variant": "mtp", "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "target_label": "unsloth/gemma-4-26B-A4B-it-GGUF", "draft": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/mtp-gemma-4-26B-A4B-it.gguf", "draft_label": "unsloth/gemma-4-26B-A4B-it-GGUF/mtp-gemma-4-26B-A4B-it.gguf" }, "security": null, "topology": null, "runtime": null, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in a Linux shell. llama-server downloads and caches the target and MTP draft GGUFs from Hugging Face.", "command": "llama-server -hf \"unsloth/gemma-4-26B-A4B-it-GGUF\" \\\n -hff \"gemma-4-26B-A4B-it-UD-Q4_K_M.gguf\" \\\n --no-mmproj \\\n -hfrd \"unsloth/gemma-4-26B-A4B-it-GGUF\" \\\n --backend-sampling \\\n --batch-size 2048 \\\n --ubatch-size 2048 \\\n --ctx-size 65536 \\\n --parallel 1 \\\n --spec-type draft-mtp \\\n --spec-draft-n-max 2 \\\n --spec-draft-model \"mtp-gemma-4-26B-A4B-it.gguf\" \\\n --temp 1.0 \\\n --top-p 0.95 \\\n --top-k 64", "port": null, "api_model": "unsloth/gemma-4-26B-A4B-it-GGUF" }, "launch": null, "sampling": "Target backend sampling: ON; speculative draft backend sampling: default; --temp=1.0; --top-p=0.95; --top-k=64", "notes": "The public [target GGUF](https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf) uses the public [MTP draft GGUF](https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/mtp-gemma-4-26B-A4B-it.gguf). Use MTP depth 2.", "clients": null }, { "id": "google/gemma-4-26b-a4b/rtx-pro/linux/vllm/nvfp4/mtp", "hardware": "rtx-pro", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.30.0", "backend_min_version": null, "min_vidmem_gb": 96, "precision": "nvfp4", "variant": "mtp", "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Gemma-4-26B-A4B-NVFP4", "target_label": "nvidia/Gemma-4-26B-A4B-NVFP4", "draft": "https://huggingface.co/google/gemma-4-26B-A4B-it-assistant", "draft_label": "google/gemma-4-26B-A4B-it-assistant" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× RTX PRO 6000", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.30.0", "image_url": "https://hub.docker.com/layers/vllm/vllm-openai/v0.30.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.30.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Gemma-4-26B-A4B-NVFP4 \\\n --max-model-len 262144 \\\n --gpu-memory-utilization 0.92 \\\n --port 8000 \\\n --host 0.0.0.0 \\\n --max-num-seqs 8 \\\n --load-format instanttensor \\\n --enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --reasoning-parser gemma4 \\\n --max-num-batched-tokens 8192 \\\n --speculative-config '{\"method\":\"mtp\",\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4, \"moe_backend\": \"triton\"}'", "port": 8000, "api_model": "nvidia/Gemma-4-26B-A4B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Gemma-4-26B-A4B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Gemma-4-26B-A4B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "google/gemma-4-26b-a4b/rtx-pro/windows/llama-cpp/24gb/q4-k-m/vision-mtp", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11256", "min_vidmem_gb": 24, "precision": "q4-k-m", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "target_label": "unsloth/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "draft": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/mtp-gemma-4-26B-A4B-it.gguf", "draft_label": "unsloth/gemma-4-26B-A4B-it-GGUF/mtp-gemma-4-26B-A4B-it.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named checkpoint files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/gemma-4-26B-A4B-it-GGUF\" `\n -hff \"gemma-4-26B-A4B-it-UD-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/resolve/c099eb48e663fd284577b04978a94ffccb261841/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 1024 `\n --ctx-size 189440 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 3 `\n -hfrd \"unsloth/gemma-4-26B-A4B-it-GGUF\" `\n --spec-draft-model \"mtp-gemma-4-26B-A4B-it.gguf\" `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 64", "port": 9931, "api_model": "unsloth/gemma-4-26B-A4B-it-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=64", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/c099eb48e663fd284577b04978a94ffccb261841/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/gemma-4-26B-A4B-it-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/gemma-4-26B-A4B-it-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "google/gemma-4-26b-a4b/rtx-pro/windows/llama-cpp/26gb/q4-k-m/vision-mtp", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11256", "min_vidmem_gb": 26, "precision": "q4-k-m", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "target_label": "unsloth/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "draft": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/mtp-gemma-4-26B-A4B-it.gguf", "draft_label": "unsloth/gemma-4-26B-A4B-it-GGUF/mtp-gemma-4-26B-A4B-it.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named checkpoint files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/gemma-4-26B-A4B-it-GGUF\" `\n -hff \"gemma-4-26B-A4B-it-UD-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/resolve/c099eb48e663fd284577b04978a94ffccb261841/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 1024 `\n --ctx-size 262144 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 3 `\n -hfrd \"unsloth/gemma-4-26B-A4B-it-GGUF\" `\n --spec-draft-model \"mtp-gemma-4-26B-A4B-it.gguf\" `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 64", "port": 9931, "api_model": "unsloth/gemma-4-26B-A4B-it-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=64", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/c099eb48e663fd284577b04978a94ffccb261841/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/gemma-4-26B-A4B-it-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/gemma-4-26B-A4B-it-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "google/gemma-4-26b-a4b/rtx-spark/windows/llama-cpp/24gb/q4-k-m/vision-mtp", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11572", "backend_min_version": "b11229", "min_vidmem_gb": 24, "precision": "q4-k-m", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build b11229, source commit c2a9e1606807970f4ee3167bacd951699c89caea, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "target_label": "unsloth/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "draft": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/mtp-gemma-4-26B-A4B-it.gguf", "draft_label": "unsloth/gemma-4-26B-A4B-it-GGUF/mtp-gemma-4-26B-A4B-it.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11572. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11572\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and draft files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/gemma-4-26B-A4B-it-GGUF\" `\n -hff \"gemma-4-26B-A4B-it-UD-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/resolve/c099eb48e663fd284577b04978a94ffccb261841/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 2048 `\n --ubatch-size 1024 `\n --ctx-size 212992 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 3 `\n -hfrd \"unsloth/gemma-4-26B-A4B-it-GGUF\" `\n --spec-draft-model \"mtp-gemma-4-26B-A4B-it.gguf\" `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 64", "port": 9931, "api_model": "unsloth/gemma-4-26B-A4B-it-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=64", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/c099eb48e663fd284577b04978a94ffccb261841/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/gemma-4-26B-A4B-it-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/gemma-4-26B-A4B-it-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "google/gemma-4-26b-a4b/rtx-spark/windows/llama-cpp/26gb/q4-k-m/vision-mtp", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11572", "backend_min_version": "b11229", "min_vidmem_gb": 26, "precision": "q4-k-m", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build b11229, source commit c2a9e1606807970f4ee3167bacd951699c89caea, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "target_label": "unsloth/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "draft": "https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/main/mtp-gemma-4-26B-A4B-it.gguf", "draft_label": "unsloth/gemma-4-26B-A4B-it-GGUF/mtp-gemma-4-26B-A4B-it.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11572. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11572\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and draft files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/gemma-4-26B-A4B-it-GGUF\" `\n -hff \"gemma-4-26B-A4B-it-UD-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/resolve/c099eb48e663fd284577b04978a94ffccb261841/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 2048 `\n --ubatch-size 1024 `\n --ctx-size 262144 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 3 `\n -hfrd \"unsloth/gemma-4-26B-A4B-it-GGUF\" `\n --spec-draft-model \"mtp-gemma-4-26B-A4B-it.gguf\" `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 64", "port": 9931, "api_model": "unsloth/gemma-4-26B-A4B-it-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=64", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/blob/c099eb48e663fd284577b04978a94ffccb261841/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/gemma-4-26B-A4B-it-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/gemma-4-26B-A4B-it-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "google/gemma-4-31b", "display_name": "Gemma-4-31B", "builder": "google", "family": "Gemma 4", "summary": "Google Gemma 4 (31B) text-generation model.", "upstream": null, "modalities": [ "text" ], "recipes": [ { "id": "google/gemma-4-31b/dgx-spark/linux/vllm/128gb/nvfp4", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.32.0rc1.dev301+g1ac942555.d20261016", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "nvfp4", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Gemma-4-31B-IT-NVFP4", "target_label": "nvidia/Gemma-4-31B-IT-NVFP4", "draft": "https://huggingface.co/google/gemma-4-31B-it-assistant", "draft_label": "google/gemma-4-31B-it-assistant" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container (community, ARM64)", "image": "eugr/spark-vllm:nightly-20261016", "image_url": "https://hub.docker.com/layers/eugr/spark-vllm/nightly-20261016", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n eugr/spark-vllm:nightly-20261016" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Gemma-4-31B-IT-NVFP4 \\\n --gpu-memory-utilization 0.80 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --enable-prefix-caching \\\n --max-num-batched-tokens 4096 \\\n --reasoning-parser gemma4 \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --load-format fastsafetensors \\\n --speculative-config '{\"model\": \"google/gemma-4-31B-it-assistant\", \"num_speculative_tokens\": 4}'", "port": 8000, "api_model": "nvidia/Gemma-4-31B-IT-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Gemma-4-31B-IT-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Gemma-4-31B-IT-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "google/gemma-4-31b/dgx-spark/linux/vllm/64gb/nvfp4", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.32.0rc1.dev301+g1ac942555.d20261016", "backend_min_version": null, "min_vidmem_gb": 64, "precision": "nvfp4", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Gemma-4-31B-IT-NVFP4", "target_label": "nvidia/Gemma-4-31B-IT-NVFP4", "draft": "https://huggingface.co/google/gemma-4-31B-it-assistant", "draft_label": "google/gemma-4-31B-it-assistant" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container (community, ARM64)", "image": "eugr/spark-vllm:nightly-20261016", "image_url": "https://hub.docker.com/layers/eugr/spark-vllm/nightly-20261016", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n eugr/spark-vllm:nightly-20261016" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Gemma-4-31B-IT-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --gpu-memory-utilization 0.80 \\\n --max-model-len 262144 \\\n --max-num-seqs 4 \\\n --enable-prefix-caching \\\n --max-num-batched-tokens 4096 \\\n --reasoning-parser gemma4 \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --load-format safetensors \\\n --speculative-config '{\"model\": \"google/gemma-4-31B-it-assistant\", \"num_speculative_tokens\": 4}'", "port": 8000, "api_model": "nvidia/Gemma-4-31B-IT-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Gemma-4-31B-IT-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Gemma-4-31B-IT-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "meta/muse-glimmer-30b", "display_name": "Muse Glimmer 30B", "builder": "meta", "family": "Muse Glimmer", "summary": "Meta Muse Glimmer 30B text-generation model.", "upstream": "https://huggingface.co/meta-models/Muse-Glimmer-30B", "modalities": [ "text", "image" ], "recipes": [ { "id": "meta/muse-glimmer-30b/dgx-station/linux/vllm/nvfp4-w4a16/dflash", "hardware": "dgx-station", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": null, "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4-w4a16", "variant": "dflash", "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Muse-Glimmer-30B-NVFP4", "target_label": "nvidia/Muse-Glimmer-30B-NVFP4", "draft": "https://huggingface.co/meta-models/Muse-Glimmer-30B-assistant", "draft_label": "meta-models/Muse-Glimmer-30B-assistant" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Station", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.31.0", "image_url": "https://hub.docker.com/layers/vllm/vllm-openai/v0.31.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --privileged \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.31.0" }, "serve": { "label": "Launch DFlash · n=15", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Muse-Glimmer-30B-NVFP4 \\\n --generation-config auto \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --enable-prefix-caching \\\n --tool-call-parser muse_glimmer \\\n --reasoning-parser muse_glimmer \\\n --speculative-config '{\"method\": \"dflash\", \"model\": \"meta-models/Muse-Glimmer-30B-assistant\", \"num_speculative_tokens\": 15}'", "port": 8000, "api_model": "nvidia/Muse-Glimmer-30B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Muse-Glimmer-30B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Muse-Glimmer-30B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "meta/muse-glimmer-30b/geforce-rtx/linux/llama-cpp/q4-k-m/dflash", "hardware": "geforce-rtx", "platform": { "os": "linux", "execution": "native" }, "backend": "llama-cpp", "backend_version": null, "backend_min_version": "b10505", "min_vidmem_gb": 20.5, "precision": "q4-k-m", "variant": "dflash", "status": "experimental", "prerequisites": [ "A Linux build of llama.cpp commit ee4c505a4fb37be8ea37a78af272e74dad2835c1 providing llama-server." ], "limitations": [ "This text-only recipe intentionally does not load the separate perception projector.", "These settings are ported from the Windows selection and have not been performance-validated on Linux.", "The source sweep used a 40,960-token server context; the launch command uses a 65,536-token context." ], "checkpoint": { "target": "https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/70bf1b61ac09f91b24d39038091b41c582bc5d7a/Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf", "target_label": "meta-models/Muse-Glimmer-30B-GGUF", "draft": "https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/70bf1b61ac09f91b24d39038091b41c582bc5d7a/dflash-Muse-Glimmer-30B-Q4_K_M.gguf", "draft_label": "meta-models/Muse-Glimmer-30B-GGUF/dflash-Muse-Glimmer-30B-Q4_K_M.gguf" }, "security": null, "topology": null, "runtime": null, "serve": { "label": "Launch DFlash · n=15", "target": "single-node", "instructions": "Run in a Linux shell. llama-server downloads and caches the target and DFlash GGUF files from Hugging Face.", "command": "llama-server -hf \"meta-models/Muse-Glimmer-30B-GGUF\" \\\n -hff \"Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf\" \\\n -hfrd \"meta-models/Muse-Glimmer-30B-GGUF\" \\\n --spec-draft-model \"dflash-Muse-Glimmer-30B-Q4_K_M.gguf\" \\\n --no-mmproj \\\n --batch-size 4096 \\\n --ubatch-size 1024 \\\n --ctx-size 65536 \\\n --parallel 1 \\\n --spec-type draft-dflash \\\n --spec-draft-n-max 15 \\\n --spec-draft-backend-sampling \\\n --temp 1.0 \\\n --top-p 0.95 \\\n --top-k 64", "port": null, "api_model": "meta-models/Muse-Glimmer-30B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=64", "notes": "The pinned [Q4_K_M target](https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/70bf1b61ac09f91b24d39038091b41c582bc5d7a/Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf) and [DFlash draft](https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/70bf1b61ac09f91b24d39038091b41c582bc5d7a/dflash-Muse-Glimmer-30B-Q4_K_M.gguf) are the selected checkpoints.", "clients": null }, { "id": "meta/muse-glimmer-30b/geforce-rtx/windows/llama-cpp/23gb/q4-k-m/vision-dflash", "hardware": "geforce-rtx", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11256", "min_vidmem_gb": 23, "precision": "q4-k-m", "variant": "vision-dflash", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoints and projector." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/main/Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf", "target_label": "meta-models/Muse-Glimmer-30B-GGUF/Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf", "draft": "https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/main/dflash-Muse-Glimmer-30B-Q4_K_M.gguf", "draft_label": "meta-models/Muse-Glimmer-30B-GGUF/dflash-Muse-Glimmer-30B-Q4_K_M.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch DFlash · n=15", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target, draft, and projector GGUFs from Hugging Face.", "command": "llama-server.exe -hf \"meta-models/Muse-Glimmer-30B-GGUF\" `\n -hff \"Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/resolve/70bf1b61ac09f91b24d39038091b41c582bc5d7a/mmproj-Muse-Glimmer-30B-Q4_K_M.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 1024 `\n --ctx-size 131072 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 15 `\n -hfrd \"meta-models/Muse-Glimmer-30B-GGUF\" `\n --spec-draft-model \"dflash-Muse-Glimmer-30B-Q4_K_M.gguf\" `\n --no-spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 64 `\n --skip-chat-parsing", "port": 9931, "api_model": "meta-models/Muse-Glimmer-30B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: OFF; --temp=1.0; --top-p=0.95; --top-k=64", "notes": "Projector: [mmproj-Muse-Glimmer-30B-Q4_K_M.gguf](https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/70bf1b61ac09f91b24d39038091b41c582bc5d7a/mmproj-Muse-Glimmer-30B-Q4_K_M.gguf).", "clients": { "shell": "'{\"model\":\"meta-models/Muse-Glimmer-30B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"meta-models/Muse-Glimmer-30B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "meta/muse-glimmer-30b/rtx-pro/linux/llama-cpp/q4-k-m/dflash", "hardware": "rtx-pro", "platform": { "os": "linux", "execution": "native" }, "backend": "llama-cpp", "backend_version": null, "backend_min_version": "b10505", "min_vidmem_gb": 20.5, "precision": "q4-k-m", "variant": "dflash", "status": "experimental", "prerequisites": [ "A Linux build of llama.cpp commit ee4c505a4fb37be8ea37a78af272e74dad2835c1 providing llama-server." ], "limitations": [ "This text-only recipe intentionally does not load the separate perception projector.", "These settings are ported from the Windows selection and have not been performance-validated on Linux.", "The source sweep used a 40,960-token server context; the launch command uses a 65,536-token context." ], "checkpoint": { "target": "https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/70bf1b61ac09f91b24d39038091b41c582bc5d7a/Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf", "target_label": "meta-models/Muse-Glimmer-30B-GGUF", "draft": "https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/70bf1b61ac09f91b24d39038091b41c582bc5d7a/dflash-Muse-Glimmer-30B-Q4_K_M.gguf", "draft_label": "meta-models/Muse-Glimmer-30B-GGUF/dflash-Muse-Glimmer-30B-Q4_K_M.gguf" }, "security": null, "topology": null, "runtime": null, "serve": { "label": "Launch DFlash · n=15", "target": "single-node", "instructions": "Run in a Linux shell. llama-server downloads and caches the target and DFlash GGUF files from Hugging Face.", "command": "llama-server -hf \"meta-models/Muse-Glimmer-30B-GGUF\" \\\n -hff \"Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf\" \\\n -hfrd \"meta-models/Muse-Glimmer-30B-GGUF\" \\\n --spec-draft-model \"dflash-Muse-Glimmer-30B-Q4_K_M.gguf\" \\\n --no-mmproj \\\n --batch-size 2048 \\\n --ubatch-size 1024 \\\n --ctx-size 65536 \\\n --parallel 1 \\\n --spec-type draft-dflash \\\n --spec-draft-n-max 15 \\\n --spec-draft-backend-sampling \\\n --temp 1.0 \\\n --top-p 0.95 \\\n --top-k 64", "port": null, "api_model": "meta-models/Muse-Glimmer-30B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=64", "notes": "The pinned [Q4_K_M target](https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/70bf1b61ac09f91b24d39038091b41c582bc5d7a/Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf) and [DFlash draft](https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/70bf1b61ac09f91b24d39038091b41c582bc5d7a/dflash-Muse-Glimmer-30B-Q4_K_M.gguf) are the selected checkpoints.", "clients": null }, { "id": "meta/muse-glimmer-30b/rtx-pro/linux/vllm/nvfp4-w4a16/dflash", "hardware": "rtx-pro", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": null, "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4-w4a16", "variant": "dflash", "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Muse-Glimmer-30B-NVFP4", "target_label": "nvidia/Muse-Glimmer-30B-NVFP4", "draft": "https://huggingface.co/meta-models/Muse-Glimmer-30B-assistant", "draft_label": "meta-models/Muse-Glimmer-30B-assistant" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× RTX Pro", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.28.0-ubuntu2404", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.28.0-ubuntu2404", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --privileged \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.28.0-ubuntu2404" }, "serve": { "label": "Launch DFlash · n=15", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Muse-Glimmer-30B-NVFP4 \\\n --generation-config auto \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --enable-prefix-caching \\\n --no-enable-flashinfer-autotune \\\n --tool-call-parser muse_glimmer \\\n --reasoning-parser muse_glimmer \\\n --speculative-config '{\"method\": \"dflash\", \"model\": \"meta-models/Muse-Glimmer-30B-assistant\", \"num_speculative_tokens\": 15}'", "port": 8000, "api_model": "nvidia/Muse-Glimmer-30B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Muse-Glimmer-30B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Muse-Glimmer-30B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "meta/muse-glimmer-30b/rtx-pro/windows/llama-cpp/24gb/q4-k-m/vision-dflash", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11256", "min_vidmem_gb": 24, "precision": "q4-k-m", "variant": "vision-dflash", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/main/Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf", "target_label": "meta-models/Muse-Glimmer-30B-GGUF/Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf", "draft": "https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/main/dflash-Muse-Glimmer-30B-Q4_K_M.gguf", "draft_label": "meta-models/Muse-Glimmer-30B-GGUF/dflash-Muse-Glimmer-30B-Q4_K_M.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch DFlash · n=15", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named checkpoint files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"meta-models/Muse-Glimmer-30B-GGUF\" `\n -hff \"Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/resolve/70bf1b61ac09f91b24d39038091b41c582bc5d7a/mmproj-Muse-Glimmer-30B-Q4_K_M.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 2048 `\n --ctx-size 121856 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 15 `\n -hfrd \"meta-models/Muse-Glimmer-30B-GGUF\" `\n --spec-draft-model \"dflash-Muse-Glimmer-30B-Q4_K_M.gguf\" `\n --no-spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 64 `\n --skip-chat-parsing", "port": 9931, "api_model": "meta-models/Muse-Glimmer-30B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: OFF; --temp=1.0; --top-p=0.95; --top-k=64", "notes": "Projector: [mmproj-Muse-Glimmer-30B-Q4_K_M.gguf](https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/70bf1b61ac09f91b24d39038091b41c582bc5d7a/mmproj-Muse-Glimmer-30B-Q4_K_M.gguf).", "clients": { "shell": "'{\"model\":\"meta-models/Muse-Glimmer-30B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"meta-models/Muse-Glimmer-30B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "meta/muse-glimmer-30b/rtx-pro/windows/llama-cpp/25gb/q4-k-m/vision-dflash", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11256", "min_vidmem_gb": 25, "precision": "q4-k-m", "variant": "vision-dflash", "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/main/Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf", "target_label": "meta-models/Muse-Glimmer-30B-GGUF/Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf", "draft": "https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/main/dflash-Muse-Glimmer-30B-Q4_K_M.gguf", "draft_label": "meta-models/Muse-Glimmer-30B-GGUF/dflash-Muse-Glimmer-30B-Q4_K_M.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch DFlash · n=15", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH.", "command": "llama-server.exe -hf \"meta-models/Muse-Glimmer-30B-GGUF\" `\n -hff \"Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/resolve/70bf1b61ac09f91b24d39038091b41c582bc5d7a/mmproj-Muse-Glimmer-30B-Q4_K_M.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 2048 `\n --ctx-size 131072 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 15 `\n -hfrd \"meta-models/Muse-Glimmer-30B-GGUF\" `\n --spec-draft-model \"dflash-Muse-Glimmer-30B-Q4_K_M.gguf\" `\n --no-spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 64 `\n --skip-chat-parsing", "port": 9931, "api_model": "meta-models/Muse-Glimmer-30B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: OFF; --temp=1.0; --top-p=0.95; --top-k=64", "notes": "Projector: [mmproj-Muse-Glimmer-30B-Q4_K_M.gguf](https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/70bf1b61ac09f91b24d39038091b41c582bc5d7a/mmproj-Muse-Glimmer-30B-Q4_K_M.gguf).", "clients": { "shell": "'{\"model\":\"meta-models/Muse-Glimmer-30B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"meta-models/Muse-Glimmer-30B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "meta/muse-glimmer-30b/rtx-spark/windows/llama-cpp/24gb/q4-k-m/vision-dflash", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11572", "backend_min_version": "b11236", "min_vidmem_gb": 24, "precision": "q4-k-m", "variant": "vision-dflash", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build b11236, source commit f1ea206218210afb913ae2f5d2c51faed35915da, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/main/Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf", "target_label": "meta-models/Muse-Glimmer-30B-GGUF/Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf", "draft": "https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/main/dflash-Muse-Glimmer-30B-Q4_K_M.gguf", "draft_label": "meta-models/Muse-Glimmer-30B-GGUF/dflash-Muse-Glimmer-30B-Q4_K_M.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11572. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11572\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch DFlash · n=15", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and draft files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"meta-models/Muse-Glimmer-30B-GGUF\" `\n -hff \"Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/resolve/70bf1b61ac09f91b24d39038091b41c582bc5d7a/mmproj-Muse-Glimmer-30B-Q4_K_M.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 2048 `\n --ubatch-size 512 `\n --ctx-size 131072 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 15 `\n -hfrd \"meta-models/Muse-Glimmer-30B-GGUF\" `\n --spec-draft-model \"dflash-Muse-Glimmer-30B-Q4_K_M.gguf\" `\n --no-spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 64", "port": 9931, "api_model": "meta-models/Muse-Glimmer-30B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: OFF; --temp=1.0; --top-p=0.95; --top-k=64", "notes": "Projector: [mmproj-Muse-Glimmer-30B-Q4_K_M.gguf](https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/blob/70bf1b61ac09f91b24d39038091b41c582bc5d7a/mmproj-Muse-Glimmer-30B-Q4_K_M.gguf).", "clients": { "shell": "'{\"model\":\"meta-models/Muse-Glimmer-30B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"meta-models/Muse-Glimmer-30B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "minimax/minimax-m2.7", "display_name": "MiniMax-M2.7", "builder": "minimax", "family": "MiniMax-M2", "summary": "MiniMax M2.7 text-generation model.", "upstream": null, "modalities": [ "text" ], "recipes": [ { "id": "minimax/minimax-m2.7/dgx-spark-2gpu/linux/vllm/nvfp4", "hardware": "dgx-spark-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21818+g3a947437c.d20261013", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "nvfp4", "variant": null, "status": "optimized", "prerequisites": [ "Docker with NVIDIA GPU access and Hugging Face access to the checkpoint on both nodes.", "Configure the ConnectX interfaces and cache the image and checkpoint on both nodes before launch." ], "limitations": [ "MTP is not supported by this configuration.", "Published coding performance was measured at concurrency 1." ], "checkpoint": { "target": "https://huggingface.co/nvidia/MiniMax-M2.7-NVFP4", "target_label": "nvidia/MiniMax-M2.7-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× DGX Spark; one node rank per Spark; tensor parallel 2", "gpu_count": 2, "node_count": 2, "tensor_parallel_size": 2 }, "runtime": { "kind": "container", "label": "Container (community, mutable, ARM64)", "image": "eugr/spark-vllm:nightly-20261013", "image_url": "https://hub.docker.com/r/eugr/spark-vllm/tags?name=nightly-20261013", "setup_label": "Start container", "setup_target": "all-nodes", "setup_instructions": "Run on every node after setting `HF_TOKEN`; replace the node-specific placeholders.", "setup_command": "# Example two-Spark values (adjust to your network):\n# HEAD_IP=\"10.100.32.1\" # head node IP on the ETH subnet\n# ETH_IF=\"enp1s0f0np0\" # ConnectX baseboard ethernet (socket/control)\n# IB_IF=\"rocep1s0f0,roceP2p1s0f0\" # RoCE HCA names (data plane) - use both \"twins\"!\nexport HEAD_IP=\"\"\nexport ETH_IF=\"\"\nexport IB_IF=\",\"\nexport MASTER_PORT=29501\nexport NODE_RANK=\"<0-1>\"\nexport IMAGE=\"eugr/spark-vllm:nightly-20261013\"\nexport HF_CACHE_DIR=\"$HOME/.cache/huggingface\"\n\nmkdir -p \"$HF_CACHE_DIR\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\"\n\ndocker run --rm -it \\\n --privileged \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit nofile=1048576:1048576 \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e VLLM_FLOAT32_MATMUL_PRECISION=high \\\n -e NODE_RANK=\"$NODE_RANK\" \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HF_CACHE_DIR:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n --entrypoint /bin/bash \\\n \"$IMAGE\"" }, "serve": { "label": "Launch head", "target": "head", "instructions": null, "command": "vllm serve nvidia/MiniMax-M2.7-NVFP4 \\\n --trust-remote-code \\\n --port 8000 \\\n --host 0.0.0.0 \\\n --gpu-memory-utilization 0.85 \\\n --kv-cache-dtype fp8 \\\n --max-model-len 196608 \\\n --load-format instanttensor \\\n --enable-auto-tool-choice \\\n --tool-call-parser minimax_m2 \\\n --reasoning-parser minimax_m2 \\\n --tensor-parallel-size 2 \\\n --nnodes 2 \\\n --node-rank 0 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\"", "port": 8000, "api_model": "nvidia/MiniMax-M2.7-NVFP4", "additional_commands": [ { "id": "worker", "label": "Launch worker", "target": "workers", "instructions": null, "command": "vllm serve nvidia/MiniMax-M2.7-NVFP4 \\\n --trust-remote-code \\\n --port 8000 \\\n --host 0.0.0.0 \\\n --gpu-memory-utilization 0.85 \\\n --kv-cache-dtype fp8 \\\n --max-model-len 196608 \\\n --load-format instanttensor \\\n --enable-auto-tool-choice \\\n --tool-call-parser minimax_m2 \\\n --reasoning-parser minimax_m2 \\\n --tensor-parallel-size 2 \\\n --nnodes 2 \\\n --node-rank 1 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\" \\\n --headless" } ] }, "launch": null, "sampling": null, "notes": "Use the same CX7 port on every node; assign both port subinterfaces on separate subnets; cache the image and checkpoint on every node; start workers before the head. Docker runs privileged.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/MiniMax-M2.7-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/MiniMax-M2.7-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "minimax/minimax-m2.7/dgx-station/linux/vllm/nvfp4", "hardware": "dgx-station", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": null, "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4", "variant": null, "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/MiniMax-M2.7-NVFP4", "target_label": "nvidia/MiniMax-M2.7-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Station", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.31.0", "image_url": "https://hub.docker.com/layers/vllm/vllm-openai/v0.31.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.31.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/MiniMax-M2.7-NVFP4 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --gpu-memory-utilization 0.92 \\\n --tensor-parallel-size 1 \\\n --tool-call-parser minimax_m2 \\\n --enable-auto-tool-choice \\\n --reasoning-parser minimax_m2 \\\n --load-format fastsafetensors \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching", "port": null, "api_model": "nvidia/MiniMax-M2.7-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": null }, { "id": "minimax/minimax-m2.7/rtx-pro-2gpu/linux/vllm/nvfp4", "hardware": "rtx-pro-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.24.0", "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/MiniMax-M2.7-NVFP4", "target_label": "nvidia/MiniMax-M2.7-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× RTX PRO 6000; tensor parallel 2", "gpu_count": 2, "node_count": null, "tensor_parallel_size": 2 }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.24.0-ubuntu2404", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.24.0-ubuntu2404", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.24.0-ubuntu2404" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/MiniMax-M2.7-NVFP4 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --gpu-memory-utilization 0.95 \\\n --tensor-parallel-size 2 \\\n --tool-call-parser minimax_m2 \\\n --enable-auto-tool-choice \\\n --reasoning-parser minimax_m2 \\\n --load-format fastsafetensors \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching", "port": null, "api_model": "nvidia/MiniMax-M2.7-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": null } ] }, { "id": "minimax/minimax-m3", "display_name": "MiniMax-M3", "builder": "minimax", "family": "MiniMax-M3", "summary": "MiniMax-M3 inference configurations.", "upstream": null, "modalities": [ "text" ], "recipes": [ { "id": "minimax/minimax-m3/dgx-spark-4gpu/linux/vllm/128gb/nvfp4", "hardware": "dgx-spark-4gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21818+g3a947437c.d20261013", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "nvfp4", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [ "The source reports this recipe currently runs out of memory with recent builds." ], "checkpoint": { "target": "https://huggingface.co/nvidia/MiniMax-M3-NVFP4", "target_label": "nvidia/MiniMax-M3-NVFP4", "draft": "https://huggingface.co/Inferact/MiniMax-M3-EAGLE3", "draft_label": "Inferact/MiniMax-M3-EAGLE3" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "4× DGX Spark; one node rank per Spark; tensor parallel 4", "gpu_count": 4, "node_count": 4, "tensor_parallel_size": 4 }, "runtime": { "kind": "container", "label": "Container (community, mutable, ARM64)", "image": "eugr/spark-vllm:nightly-20261013", "image_url": "https://hub.docker.com/r/eugr/spark-vllm/tags?name=nightly-20261013", "setup_label": "Start container", "setup_target": "all-nodes", "setup_instructions": "Run on every node after setting `HF_TOKEN`; replace the node-specific placeholders.", "setup_command": "# Example four-Spark values (adjust to your network):\n# HEAD_IP=\"10.100.32.1\" # head node IP on the ETH subnet\n# ETH_IF=\"enp1s0f0np0\" # ConnectX baseboard ethernet (socket/control)\n# IB_IF=\"rocep1s0f0,roceP2p1s0f0\" # RoCE HCA names (data plane) - use both \"twins\"!\nexport HEAD_IP=\"\"\nexport ETH_IF=\"\"\nexport IB_IF=\",\"\nexport MASTER_PORT=29501\nexport NODE_RANK=\"<0-3>\"\nexport IMAGE=\"eugr/spark-vllm:nightly-20261013\"\nexport HF_CACHE_DIR=\"$HOME/.cache/huggingface\"\n\nmkdir -p \"$HF_CACHE_DIR\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\"\n\ndocker run --rm -it \\\n --privileged \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit nofile=1048576:1048576 \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e VLLM_FLOAT32_MATMUL_PRECISION=high \\\n -e NODE_RANK=\"$NODE_RANK\" \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HF_CACHE_DIR:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n --entrypoint /bin/bash \\\n \"$IMAGE\"" }, "serve": { "label": "Launch head", "target": "head", "instructions": null, "command": "vllm serve nvidia/MiniMax-M3-NVFP4 \\\n --tensor-parallel-size 4 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --gpu-memory-utilization 0.87 \\\n --kv-cache-dtype fp8 \\\n --moe-backend flashinfer_b12x \\\n --load-format instanttensor \\\n --block-size 128 \\\n --max-model-len 262144 \\\n --attention-backend triton_attn \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice \\\n --default-chat-template-kwargs '{\"thinking_mode\":\"enabled\"}' \\\n --speculative-config '{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"TRITON_ATTN\"}' \\\n --nnodes 4 \\\n --node-rank 0 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\"", "port": null, "api_model": "nvidia/MiniMax-M3-NVFP4", "additional_commands": [ { "id": "workers", "label": "Launch workers", "target": "workers", "instructions": null, "command": "vllm serve nvidia/MiniMax-M3-NVFP4 \\\n --tensor-parallel-size 4 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --gpu-memory-utilization 0.87 \\\n --kv-cache-dtype fp8 \\\n --moe-backend flashinfer_b12x \\\n --load-format instanttensor \\\n --block-size 128 \\\n --max-model-len 262144 \\\n --attention-backend triton_attn \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice \\\n --default-chat-template-kwargs '{\"thinking_mode\":\"enabled\"}' \\\n --speculative-config '{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"TRITON_ATTN\"}' \\\n --nnodes 4 \\\n --node-rank \"$NODE_RANK\" \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\" \\\n --headless" } ] }, "launch": null, "sampling": null, "notes": "Use the same CX7 port on every node; assign both port subinterfaces on separate subnets; cache the image and checkpoint on every node; start workers before the head. Docker runs privileged.", "clients": null } ] }, { "id": "minimax/minimax-m3.1-flash", "display_name": "MiniMax-M3.1-Flash", "builder": "minimax", "family": "MiniMax-M3", "summary": "MiniMax-M3.1-Flash inference for text, image, and video inputs.", "upstream": "https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash", "modalities": [ "text", "image", "video" ], "recipes": [ { "id": "minimax/minimax-m3.1-flash/dgx-spark-4gpu/linux/vllm/fp4-fp8-mixed", "hardware": "dgx-spark-4gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev23341+g8744dd2fb.d20261022.cu134", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "fp4-fp8-mixed", "variant": null, "status": "experimental", "prerequisites": [ "Four 128 GB DGX Spark nodes with Docker and a configured ConnectX RoCE network." ], "limitations": [ "Image and video inputs and requests near the configured context limit have not been validated." ], "checkpoint": { "target": "https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash", "target_label": "MiniMaxAI/MiniMax-M3.1-Flash", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "4× DGX Spark; 128 GB per node; one node rank per Spark; tensor parallel 4", "gpu_count": 4, "node_count": 4, "tensor_parallel_size": 4 }, "runtime": { "kind": "container", "label": "Container (official, mutable, ARM64)", "image": "vllm/vllm-openai:minimax-m31-flash", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=minimax-m31-flash", "setup_label": "Start container", "setup_target": "all-nodes", "setup_instructions": "Run on all four nodes. Use the same head node ConnectX address (`HEAD_IP`) and rendezvous port (`MASTER_PORT`) on every node. Set `ETH_IF` to the local ConnectX Ethernet interface for control traffic and `IB_IF` to both local RoCE host channel adapter names for data traffic.", "setup_command": "# Example four-Spark network values; adjust to the configured network:\n# HEAD_IP=\"10.100.32.1\" # head node's ConnectX address\n# ETH_IF=\"enp1s0f0np0\" # local Ethernet interface for control traffic\n# IB_IF=\"rocep1s0f0,roceP2p1s0f0\" # both local RoCE adapters for data traffic\nexport HEAD_IP=\"\"\nexport ETH_IF=\"\"\nexport IB_IF=\",\"\nexport MASTER_PORT=29501\nexport IMAGE=\"vllm/vllm-openai:minimax-m31-flash\"\nexport HF_CACHE_DIR=\"$HOME/.cache/huggingface\"\n\nmkdir -p \"$HF_CACHE_DIR\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\"\n\ndocker run --rm -it \\\n --privileged \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit nofile=1048576:1048576 \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e NCCL_MAX_CTAS=8 \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HF_CACHE_DIR:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n --entrypoint /bin/bash \\\n \"$IMAGE\"" }, "serve": { "label": "Launch head", "target": "head", "instructions": "Run inside the head container after starting all three workers.", "command": "vllm serve MiniMaxAI/MiniMax-M3.1-Flash \\\n --gpu-memory-utilization 0.8 \\\n --max-model-len 262144 \\\n --max-num-seqs 2 \\\n --attention-config '{\"indexer_kv_dtype\":\"fp8\",\"minimax_m3_msa_decode_backend\":\"cutlass\"}' \\\n --kv-cache-dtype fp8 \\\n --enable-auto-tool-choice \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --tensor-parallel-size 4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_tensor_parallel_size\":4, \"kv_cache_dtype\":\"bfloat16\"}' \\\n --load-format instanttensor \\\n --nnodes 4 \\\n --node-rank 0 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\"", "port": null, "api_model": "MiniMaxAI/MiniMax-M3.1-Flash", "additional_commands": [ { "id": "workers", "label": "Launch workers", "target": "workers", "instructions": "Run inside each worker container. Set `NODE_RANK` to a unique value of 1, 2, or 3; rank 0 belongs to the head.", "command": "export NODE_RANK=\"\"\n\nvllm serve MiniMaxAI/MiniMax-M3.1-Flash \\\n --gpu-memory-utilization 0.8 \\\n --max-model-len 262144 \\\n --max-num-seqs 2 \\\n --attention-config '{\"indexer_kv_dtype\":\"fp8\",\"minimax_m3_msa_decode_backend\":\"cutlass\"}' \\\n --kv-cache-dtype fp8 \\\n --enable-auto-tool-choice \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --tensor-parallel-size 4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_tensor_parallel_size\":4, \"kv_cache_dtype\":\"bfloat16\"}' \\\n --load-format instanttensor \\\n --nnodes 4 \\\n --node-rank \"$NODE_RANK\" \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\" \\\n --headless" } ] }, "launch": null, "sampling": null, "notes": "Use the same ConnectX-7 port on every node and assign its two subinterfaces to separate subnets. Cache the image and checkpoint on every node. Start all three workers before the head. Docker runs privileged. Checkpoint quantization is MXFP8 with NVFP4 MoE weights; the checkpoint config declares `torch_dtype` as `bfloat16`.", "clients": null }, { "id": "minimax/minimax-m3.1-flash/dgx-station/linux/vllm/fp4-fp8-mixed", "hardware": "dgx-station", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev23341+g8744dd2fb.d20261022.cu134", "backend_min_version": null, "min_vidmem_gb": null, "precision": "fp4-fp8-mixed", "variant": null, "status": "experimental", "prerequisites": [], "limitations": [ "Only 1 user can run with 256k context length, but 4 can be run with 64k context each." ], "checkpoint": { "target": "https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash", "target_label": "MiniMaxAI/MiniMax-M3.1-Flash", "draft": "https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash", "draft_label": "MiniMaxAI/MiniMax-M3.1-Flash" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Station", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:minimax-m31-flash", "image_url": "https://hub.docker.com/layers/vllm/vllm-openai/minimax-m31-flash", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:minimax-m31-flash" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY=1 \\\nPYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \\\nVLLM_FLOAT32_MATMUL_PRECISION=high \\\nvllm serve MiniMaxAI/MiniMax-M3.1-Flash \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --max-model-len 262144 \\\n --max-num-seqs 4 \\\n --max-num-batched-tokens 2048 \\\n --block-size 128 \\\n --kv-cache-dtype nvfp4 \\\n --kv-cache-memory-bytes 11545482000 \\\n --attention-config '{\"minimax_m3_msa_decode_backend\":\"cutlass\",\"indexer_kv_dtype\":\"nvfp4\"}' \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":3,\"attention_backend\":\"FLASH_ATTN\",\"kv_cache_dtype\":\"fp8\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"block\"}' \\\n --enable-auto-tool-choice \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --cpu-offload-gb 25 \\\n --cpu-offload-params experts \\\n --offload-backend uva \\\n --limit-mm-per-prompt '{\"image\":{\"count\":1,\"width\":384,\"height\":384},\"video\":{\"count\":1,\"num_frames\":16,\"width\":384,\"height\":384}}' \\\n --media-io-kwargs '{\"video\":{\"num_frames\":16}}'", "port": 8000, "api_model": "MiniMaxAI/MiniMax-M3.1-Flash" }, "launch": null, "sampling": null, "notes": "The model runs on a single DGX Station with 256k context length, speculative decoding and multimodal features with CPU offloading. Only 1 user can use the full context length, but 4 can be run with reduced context.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"MiniMaxAI/MiniMax-M3.1-Flash\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"MiniMaxAI/MiniMax-M3.1-Flash\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "nvidia/nemotron-3-super-120b-a12b", "display_name": "NVIDIA Nemotron 3 Super 120B A12B", "builder": "nvidia", "family": "Nemotron 3", "summary": "NVIDIA Nemotron 3 Super (120B-A12B) text-generation model.", "upstream": null, "modalities": [ "text" ], "recipes": [ { "id": "nvidia/nemotron-3-super-120b-a12b/dgx-spark/linux/vllm/nvfp4/mtp", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.31.0", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "nvfp4", "variant": "mtp", "status": "optimized", "prerequisites": [ "Docker with NVIDIA GPU access and Hugging Face access to the checkpoint." ], "limitations": [ "Published coding performance was measured at concurrency 1." ], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "target_label": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.31.0", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.31.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.31.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "VLLM_FLOAT32_MATMUL_PRECISION=\"high\" \\\nVLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \\\nvllm serve nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --max-model-len 262144 \\\n --gpu-memory-utilization 0.8 \\\n --max-num-seqs 8 \\\n --load-format fastsafetensors \\\n --attention-backend flashinfer \\\n --max-num-batched-tokens 16384 \\\n --kv-cache-dtype fp8 \\\n --mamba-cache-mode align \\\n --mamba_ssm_cache_dtype float32 \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}' \\\n --enable-prefix-caching \\\n --reasoning-parser nemotron_v3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_xml", "port": 8000, "api_model": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "nvidia/nemotron-3-super-120b-a12b/dgx-station/linux/vllm/nvfp4/mtp", "hardware": "dgx-station", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.31.0", "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4", "variant": "mtp", "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "target_label": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Station", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.31.0", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.31.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.31.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "VLLM_FLOAT32_MATMUL_PRECISION=\"high\" \\\nVLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \\\nvllm serve nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --max-model-len 262144 \\\n --gpu-memory-utilization 0.92 \\\n --load-format fastsafetensors \\\n --attention-backend flashinfer \\\n --max-num-batched-tokens 16384 \\\n --kv-cache-dtype fp8 \\\n --mamba-cache-mode align \\\n --mamba_ssm_cache_dtype float32 \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}' \\\n --enable-prefix-caching \\\n --reasoning-parser nemotron_v3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_xml", "port": 8000, "api_model": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "nvidia/nemotron-3-super-120b-a12b/rtx-pro/linux/vllm/nvfp4/mtp", "hardware": "rtx-pro", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": null, "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4", "variant": "mtp", "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "target_label": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× RTX PRO 6000", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.24.0-ubuntu2404", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.24.0-ubuntu2404", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.24.0-ubuntu2404" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "VLLM_FLOAT32_MATMUL_PRECISION=\"high\" \\\nVLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \\\nvllm serve nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --max-model-len 262144 \\\n --gpu-memory-utilization 0.92 \\\n --max-num-seqs 8 \\\n --attention-backend flashinfer \\\n --max-num-batched-tokens 16384 \\\n --kv-cache-dtype fp8 \\\n --mamba-cache-mode align \\\n --mamba_ssm_cache_dtype float32 \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}' \\\n --enable-prefix-caching \\\n --reasoning-parser nemotron_v3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_xml", "port": 8000, "api_model": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "nvidia/nemotron-3.5-bolt-74b-a9b", "display_name": "NVIDIA Nemotron 3.5 Bolt 74B A9B", "builder": "nvidia", "family": "Nemotron 3.5", "summary": "NVIDIA Nemotron 3.5 Bolt 74B A9B inference configurations.", "upstream": null, "modalities": [ "text", "image" ], "recipes": [ { "id": "nvidia/nemotron-3.5-bolt-74b-a9b/dgx-spark/linux/vllm/128gb/iq2", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.32.0rc1.dev227+g1336955f3.d20261015", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "iq2", "variant": null, "status": "optimized", "prerequisites": [ "A single DGX Spark with 128 GB of memory.", "Docker with NVIDIA GPU support and access to the target and draft checkpoints." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB", "target_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB", "draft": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr", "draft_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container (community, ARM64)", "image": "eugr/spark-vllm:nightly-20261015", "image_url": "https://hub.docker.com/layers/eugr/spark-vllm/nightly-20261015", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment. Host cache directories are created and mounted to persist Hugging Face, vLLM, FlashInfer, and Triton caches across container runs.", "setup_command": "mkdir -p \"$HOME/.cache/huggingface\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\"\n\ndocker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v \"$HOME/.cache/huggingface:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n --entrypoint /bin/bash \\\n eugr/spark-vllm:nightly-20261015" }, "serve": { "label": "Launch", "target": "single-node", "instructions": "Run inside the container started above.", "command": "vllm serve nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB \\\n --kv-cache-dtype fp8 \\\n --max-num-seqs 8 \\\n --trust-remote-code \\\n --max-model-len auto \\\n --gpu-memory-utilization 0.8 \\\n --max-num-batched-tokens 8192 \\\n --speculative_config.model nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr \\\n --speculative_config.method dflash \\\n --speculative_config.num_speculative_tokens 7 \\\n --load-format instanttensor \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser nemotron_v3 \\\n --enable-mamba-shared-prefix-checkpoint \\\n --block-size 64 \\\n --prefix-match-unit 64", "port": null, "api_model": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB" }, "launch": null, "sampling": null, "notes": null, "clients": null }, { "id": "nvidia/nemotron-3.5-bolt-74b-a9b/dgx-spark/linux/vllm/64gb/iq2", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.32.0rc1.dev301+g1ac942555.d20261016", "backend_min_version": null, "min_vidmem_gb": 64, "precision": "iq2", "variant": null, "status": "optimized", "prerequisites": [ "A single DGX Spark with 64 GB of memory.", "Docker with NVIDIA GPU support and access to the target and draft checkpoints." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB", "target_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB", "draft": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr", "draft_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container (community, ARM64)", "image": "eugr/spark-vllm:nightly-20261016", "image_url": "https://hub.docker.com/layers/eugr/spark-vllm/nightly-20261016", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment. Host cache directories are created and mounted to persist Hugging Face, vLLM, FlashInfer, and Triton caches across container runs.", "setup_command": "mkdir -p \"$HOME/.cache/huggingface\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\"\n\ndocker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v \"$HOME/.cache/huggingface:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n --entrypoint /bin/bash \\\n eugr/spark-vllm:nightly-20261016" }, "serve": { "label": "Launch", "target": "single-node", "instructions": "Run inside the container started above.", "command": "vllm serve nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB \\\n --kv-cache-dtype fp8 \\\n --max-num-seqs 8 \\\n --trust-remote-code \\\n --max-model-len auto \\\n --gpu-memory-utilization 0.8 \\\n --max-num-batched-tokens 4096 \\\n --speculative_config.model nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr \\\n --speculative_config.method dflash \\\n --speculative_config.num_speculative_tokens 7 \\\n --load-format instanttensor \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser nemotron_v3 \\\n --enable-mamba-shared-prefix-checkpoint \\\n --block-size 64 \\\n --prefix-match-unit 64", "port": null, "api_model": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB" }, "launch": null, "sampling": null, "notes": null, "clients": null }, { "id": "nvidia/nemotron-3.5-bolt-74b-a9b/geforce-rtx/windows/llama-cpp/28gb/iq2/vision-dflash", "hardware": "geforce-rtx", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11669", "backend_min_version": "b11669", "min_vidmem_gb": 28, "precision": "iq2", "variant": "vision-dflash", "status": "experimental", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11669, source commit 620d2225cc436ab0b001e3200e590e3b286ef9ab, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/blob/main/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf", "target_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf", "draft": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr-GGUF/blob/main/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr.gguf", "draft_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr-GGUF/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11669. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11669\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch LiLiCorr · n=6", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target, draft, and projector files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF\" `\n -hff \"NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf\" `\n --mmproj-url \"https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/resolve/main/mmproj-NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf\" `\n --batch-size 2048 `\n --ubatch-size 512 `\n --ctx-size 65536 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 6 `\n -hfrd \"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr-GGUF\" `\n --spec-draft-model \"NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr.gguf\" `\n --spec-draft-ngl 999 `\n --cache-type-k q8_0 `\n --cache-type-v q8_0 `\n --temp 1.0 `\n --top-p 0.95 `\n --flash-attn on `\n --jinja `\n --host 127.0.0.1 `\n --port 9931", "port": 9931, "api_model": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF" }, "launch": null, "sampling": "--temp=1.0; --top-p=0.95", "notes": "Projector: [mmproj-NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf](https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/blob/main/mmproj-NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf).", "clients": { "shell": "'{\"model\":\"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "nvidia/nemotron-3.5-bolt-74b-a9b/rtx-pro/linux/vllm/iq2", "hardware": "rtx-pro", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.32.0rc1.dev190+g3f1bcfd82", "backend_min_version": null, "min_vidmem_gb": null, "precision": "iq2", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB", "target_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB", "draft": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr", "draft_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× RTX Pro", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:nightly", "image_url": "https://hub.docker.com/layers/vllm/vllm-openai/nightly", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:nightly" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "# Install b12x==1.5.1 in the container and download the kernels\npip install b12x==1.5.1\nvllm download-kernels\n\nvllm serve \\\n --model nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB \\\n --block-size 64 \\\n --enable-auto-tool-choice \\\n --enable-mamba-cache-stochastic-rounding \\\n --enable-mamba-shared-prefix-checkpoint \\\n --mamba-backend flashinfer \\\n --mamba-cache-philox-rounds 5 \\\n --mamba-ssm-cache-dtype float16 \\\n --max-model-len 262144 \\\n --max-num-batched-tokens 8192 \\\n --max-num-seqs 8 \\\n --prefix-match-unit 64 \\\n --reasoning-parser nemotron_v3 \\\n --speculative_config.method dflash \\\n --speculative_config.model nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr \\\n --speculative_config.num_speculative_tokens 7 \\\n --tool-call-parser qwen3_coder \\\n --trust-remote-code", "port": 8000, "api_model": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "nvidia/nemotron-3.5-bolt-74b-a9b/rtx-pro/windows/llama-cpp/iq2/vision-dflash", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11669", "backend_min_version": "b11669", "min_vidmem_gb": 32, "precision": "iq2", "variant": "vision-dflash", "status": "experimental", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11669, source commit 620d2225cc436ab0b001e3200e590e3b286ef9ab, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/blob/main/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf", "target_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf", "draft": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr-GGUF/blob/main/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr.gguf", "draft_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr-GGUF/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11669. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11669\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch LiLiCorr · n=7", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target, draft, and projector files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF\" `\n -hff \"NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf\" `\n --mmproj-url \"https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/resolve/main/mmproj-NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf\" `\n --batch-size 4096 `\n --ubatch-size 2048 `\n --ctx-size 262144 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 7 `\n -hfrd \"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr-GGUF\" `\n --spec-draft-model \"NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr.gguf\" `\n --spec-draft-ngl 999 `\n --temp 1.0 `\n --top-p 0.95 `\n --flash-attn on `\n --jinja `\n --host 127.0.0.1 `\n --port 9931", "port": 9931, "api_model": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF" }, "launch": null, "sampling": "--temp=1.0; --top-p=0.95", "notes": "Projector: [mmproj-NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf](https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/blob/main/mmproj-NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf).", "clients": { "shell": "'{\"model\":\"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "nvidia/nemotron-3.5-bolt-74b-a9b/rtx-spark/windows/llama-cpp/28gb/iq2/vision-dflash", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11669", "backend_min_version": "b11669", "min_vidmem_gb": 28, "precision": "iq2", "variant": "vision-dflash", "status": "experimental", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build b11669, source commit 620d2225cc436ab0b001e3200e590e3b286ef9ab, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/blob/main/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf", "target_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf", "draft": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr-GGUF/blob/main/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr.gguf", "draft_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr-GGUF/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11669. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11669\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch LiLiCorr · n=6", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target, draft, and projector files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF\" `\n -hff \"NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf\" `\n --mmproj-url \"https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/resolve/main/mmproj-NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf\" `\n --batch-size 2048 `\n --ubatch-size 512 `\n --ctx-size 65536 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 6 `\n -hfrd \"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr-GGUF\" `\n --spec-draft-model \"NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr.gguf\" `\n --spec-draft-ngl 999 `\n --cache-type-k q8_0 `\n --cache-type-v q8_0 `\n --temp 1.0 `\n --top-p 0.95 `\n --flash-attn on `\n --jinja `\n --host 127.0.0.1 `\n --port 9931", "port": 9931, "api_model": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF" }, "launch": null, "sampling": "--temp=1.0; --top-p=0.95", "notes": "Projector: [mmproj-NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf](https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/blob/main/mmproj-NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf).", "clients": { "shell": "'{\"model\":\"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "nvidia/nemotron-3.5-bolt-74b-a9b/rtx-spark/windows/llama-cpp/32gb/iq2/vision-dflash", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11669", "backend_min_version": "b11669", "min_vidmem_gb": 32, "precision": "iq2", "variant": "vision-dflash", "status": "experimental", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build b11669, source commit 620d2225cc436ab0b001e3200e590e3b286ef9ab, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/blob/main/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf", "target_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf", "draft": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr-GGUF/blob/main/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr.gguf", "draft_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr-GGUF/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11669. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11669\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch LiLiCorr · n=7", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target, draft, and projector files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF\" `\n -hff \"NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf\" `\n --mmproj-url \"https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/resolve/main/mmproj-NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf\" `\n --batch-size 4096 `\n --ubatch-size 2048 `\n --ctx-size 262144 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 7 `\n -hfrd \"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr-GGUF\" `\n --spec-draft-model \"NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr.gguf\" `\n --spec-draft-ngl 999 `\n --temp 1.0 `\n --top-p 0.95 `\n --flash-attn on `\n --jinja `\n --host 127.0.0.1 `\n --port 9931", "port": 9931, "api_model": "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF" }, "launch": null, "sampling": "--temp=1.0; --top-p=0.95", "notes": "Projector: [mmproj-NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf](https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF/blob/main/mmproj-NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf).", "clients": { "shell": "'{\"model\":\"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "nvidia/nemotron-3.5-lightning-30b-a3b", "display_name": "NVIDIA Nemotron 3.5 Lightning 30B A3B", "builder": "nvidia", "family": "Nemotron 3.5", "summary": "NVIDIA Nemotron 3.5 Lightning 30B A3B inference configurations.", "upstream": null, "modalities": [ "text" ], "recipes": [ { "id": "nvidia/nemotron-3.5-lightning-30b-a3b/dgx-spark/linux/vllm/64gb/nvfp4", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.32.0rc1.dev301+g1ac942555.d20261016", "backend_min_version": null, "min_vidmem_gb": 64, "precision": "nvfp4", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", "target_label": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", "draft": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", "draft_label": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container (community, ARM64)", "image": "eugr/spark-vllm:nightly-20261016", "image_url": "https://hub.docker.com/layers/eugr/spark-vllm/nightly-20261016", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n eugr/spark-vllm:nightly-20261016" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 \\\n --moe-backend marlin \\\n --kv-cache-dtype fp8 \\\n --max-num-batched-tokens 4096 \\\n --gpu-memory-utilization 0.8 \\\n --enable-prefix-caching \\\n --mamba-backend flashinfer \\\n --mamba-cache-mode align \\\n --mamba-ssm-cache-dtype float16 \\\n --enable-mamba-cache-stochastic-rounding \\\n --mamba-cache-philox-rounds 5 \\\n --reasoning-parser nemotron_v3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_xml \\\n --load-format safetensors \\\n --speculative_config.method dspark \\\n --speculative_config.model nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark \\\n --speculative_config.num_speculative_tokens 7", "port": null, "api_model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": null }, { "id": "nvidia/nemotron-3.5-lightning-30b-a3b/geforce-rtx/linux/llama-cpp/q4-k-m/mtp", "hardware": "geforce-rtx", "platform": { "os": "linux", "execution": "native" }, "backend": "llama-cpp", "backend_version": null, "backend_min_version": "b10505", "min_vidmem_gb": 25.5, "precision": "q4-k-m", "variant": "mtp", "status": "experimental", "prerequisites": [ "A Linux build of llama.cpp commit fe8156f789011f6ea0baf6917ea09f88b89d9554 providing llama-server." ], "limitations": [ "These settings are ported from the Windows selection and have not been performance-validated on Linux.", "The source sweep used a 40,960-token server context; the launch command uses a 65,536-token context." ], "checkpoint": { "target": "https://huggingface.co/unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/blob/f2d3fe3694501008786e81e5f20360cbf715496a/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf", "target_label": "unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": null, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in a Linux shell. llama-server downloads and caches the integrated-MTP target GGUF from Hugging Face.", "command": "llama-server -hf \"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF\" \\\n -hff \"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf\" \\\n --no-mmproj \\\n --backend-sampling \\\n --batch-size 2048 \\\n --ubatch-size 2048 \\\n --ctx-size 65536 \\\n --parallel 1 \\\n --spec-type draft-mtp \\\n --spec-draft-n-max 3 \\\n --spec-draft-backend-sampling \\\n --temp 1.0 \\\n --top-p 0.95", "port": null, "api_model": "unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF" }, "launch": null, "sampling": "Target backend sampling: ON; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95", "notes": "The pinned [Q4_K_M target GGUF](https://huggingface.co/unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/blob/f2d3fe3694501008786e81e5f20360cbf715496a/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf) contains integrated MTP layers.", "clients": null }, { "id": "nvidia/nemotron-3.5-lightning-30b-a3b/geforce-rtx/windows/llama-cpp/30gb/q4-k-m/mtp", "hardware": "geforce-rtx", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11256", "min_vidmem_gb": 30, "precision": "q4-k-m", "variant": "mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/blob/main/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf", "target_label": "unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the integrated-MTP target checkpoint from Hugging Face.", "command": "llama-server.exe -hf \"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF\" `\n -hff \"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf\" `\n --no-mmproj `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 4096 `\n --ctx-size 201728 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 2 `\n --no-spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 40 `\n --min-p 0.05 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: OFF; --temp=1.0; --top-p=0.95; --top-k=40; --min-p=0.05; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": null, "clients": { "shell": "'{\"model\":\"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "nvidia/nemotron-3.5-lightning-30b-a3b/rtx-pro/linux/llama-cpp/q4-k-m/mtp", "hardware": "rtx-pro", "platform": { "os": "linux", "execution": "native" }, "backend": "llama-cpp", "backend_version": null, "backend_min_version": "b10505", "min_vidmem_gb": 25.5, "precision": "q4-k-m", "variant": "mtp", "status": "experimental", "prerequisites": [ "A Linux build of llama.cpp commit fe8156f789011f6ea0baf6917ea09f88b89d9554 providing llama-server." ], "limitations": [ "These settings are ported from the Windows selection and have not been performance-validated on Linux.", "The source sweep used a 40,960-token server context; the launch command uses a 65,536-token context." ], "checkpoint": { "target": "https://huggingface.co/unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/blob/f2d3fe3694501008786e81e5f20360cbf715496a/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf", "target_label": "unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": null, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in a Linux shell. llama-server downloads and caches the integrated-MTP target GGUF from Hugging Face.", "command": "llama-server -hf \"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF\" \\\n -hff \"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf\" \\\n --no-mmproj \\\n --batch-size 2048 \\\n --ubatch-size 2048 \\\n --ctx-size 65536 \\\n --parallel 1 \\\n --spec-type draft-mtp \\\n --spec-draft-n-max 3 \\\n --spec-draft-backend-sampling \\\n --temp 1.0 \\\n --top-p 0.95", "port": null, "api_model": "unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95", "notes": "The pinned [Q4_K_M target GGUF](https://huggingface.co/unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/blob/f2d3fe3694501008786e81e5f20360cbf715496a/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf) contains integrated MTP layers.", "clients": null }, { "id": "nvidia/nemotron-3.5-lightning-30b-a3b/rtx-pro/windows/llama-cpp/q4-k-m/mtp", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11256", "min_vidmem_gb": 30, "precision": "q4-k-m", "variant": "mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/blob/main/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf", "target_label": "unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the integrated-MTP target checkpoint from Hugging Face.", "command": "llama-server.exe -hf \"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF\" `\n -hff \"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf\" `\n --no-mmproj `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 2048 `\n --ctx-size 262144 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 2 `\n --no-spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 40 `\n --min-p 0.05 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: OFF; --temp=1.0; --top-p=0.95; --top-k=40; --min-p=0.05; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": null, "clients": { "shell": "'{\"model\":\"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "nvidia/nemotron-3.5-lightning-30b-a3b/rtx-spark/windows/llama-cpp/28gb/q4-k-m/mtp", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11572", "backend_min_version": "b11229", "min_vidmem_gb": 28, "precision": "q4-k-m", "variant": "mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build b11229, source commit c2a9e1606807970f4ee3167bacd951699c89caea, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/blob/main/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf", "target_label": "unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11572. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11572\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and draft files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF\" `\n -hff \"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf\" `\n --no-mmproj `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 4096 `\n --ctx-size 98304 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 2 `\n --no-spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95", "port": 9931, "api_model": "unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: OFF; --temp=1.0; --top-p=0.95", "notes": null, "clients": { "shell": "'{\"model\":\"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "nvidia/nemotron-3.5-lightning-30b-a3b/rtx-spark/windows/llama-cpp/32gb/q4-k-xl/mtp", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11572", "backend_min_version": "b11229", "min_vidmem_gb": 32, "precision": "q4-k-xl", "variant": "mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build b11229, source commit c2a9e1606807970f4ee3167bacd951699c89caea, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/blob/main/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_XL.gguf", "target_label": "unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_XL.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11572. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11572\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and draft files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF\" `\n -hff \"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_XL.gguf\" `\n --no-mmproj `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 4096 `\n --ctx-size 262144 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 2 `\n --no-spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95", "port": 9931, "api_model": "unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: OFF; --temp=1.0; --top-p=0.95", "notes": null, "clients": { "shell": "'{\"model\":\"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "nvidia/nemotron-3.5-super-vl-120b-a12b", "display_name": "NVIDIA Nemotron 3.5 Super VL 120B A12B", "builder": "nvidia", "family": "Nemotron 3.5", "summary": "NVIDIA Nemotron 3.5 Super VL 120B A12B inference configurations.", "upstream": null, "modalities": [ "text", "image", "video" ], "recipes": [ { "id": "nvidia/nemotron-3.5-super-vl-120b-a12b/dgx-spark/linux/vllm/nvfp4", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.32.0rc1.dev227+g1336955f3.d20261015", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "nvfp4", "variant": null, "status": "optimized", "prerequisites": [ "A single DGX Spark with 128 GB of memory.", "Docker with NVIDIA GPU support and access to the target and draft checkpoints." ], "limitations": [ "Text performance was measured; image and video workloads were not benchmarked." ], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4", "target_label": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4", "draft": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4-LiLiCorr", "draft_label": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4-LiLiCorr" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container (community, mutable, ARM64)", "image": "eugr/spark-vllm:latest", "image_url": "https://hub.docker.com/r/eugr/spark-vllm/tags?name=latest", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment. Host cache directories are created and mounted to persist Hugging Face, vLLM, FlashInfer, and Triton caches across container runs.", "setup_command": "mkdir -p \"$HOME/.cache/huggingface\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\"\n\ndocker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v \"$HOME/.cache/huggingface:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n --entrypoint /bin/bash \\\n eugr/spark-vllm:latest" }, "serve": { "label": "Launch", "target": "single-node", "instructions": "Run inside the container started above.", "command": "vllm serve nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4 \\\n --kv-cache-dtype fp8 \\\n --max-num-seqs 8 \\\n --trust-remote-code \\\n --max-model-len auto \\\n --gpu-memory-utilization 0.8 \\\n --max-num-batched-tokens 8192 \\\n --speculative_config.model nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4-LiLiCorr \\\n --speculative_config.method dflash \\\n --speculative_config.num_speculative_tokens 7 \\\n --mamba-backend flashinfer \\\n --mamba-ssm-cache-dtype float16 \\\n --enable-mamba-cache-stochastic-rounding \\\n --mamba-cache-philox-rounds 5 \\\n --enable-mamba-shared-prefix-checkpoint \\\n --prefix-match-unit 64 \\\n --block-size 64 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser nemotron_v3 \\\n --mm-processor-cache-gb 0 \\\n --limit-mm-per-prompt '{\"video\": 1, \"image\": 1}' \\\n --video-pruning-rate 0.5 \\\n --media-io-kwargs '{\"video\": {\"fps\": 2, \"num_frames\": 256}}' \\\n --load-format instanttensor", "port": null, "api_model": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": null }, { "id": "nvidia/nemotron-3.5-super-vl-120b-a12b/dgx-station/linux/vllm/nvfp4", "hardware": "dgx-station", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.31.0", "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4", "target_label": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4", "draft": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4-LiLiCorr", "draft_label": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4-LiLiCorr" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Station", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.31.0", "image_url": "https://hub.docker.com/layers/vllm/vllm-openai/v0.31.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.31.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "VLLM_FLOAT32_MATMUL_PRECISION=\"high\" \\\nVLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \\\nvllm serve nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4 \\\n --allowed-local-media-path / \\\n --async-scheduling \\\n --enable-auto-tool-choice \\\n --enable-mamba-cache-stochastic-rounding \\\n --enable-mamba-shared-prefix-checkpoint \\\n --gpu-memory-utilization 0.92 \\\n --kv-cache-dtype fp8 \\\n --limit-mm-per-prompt '{\"video\": 1, \"image\": 1}' \\\n --load-format instanttensor \\\n --mamba-backend flashinfer \\\n --mamba-cache-philox-rounds 5 \\\n --mamba-ssm-cache-dtype float16 \\\n --max-model-len 1048576 \\\n --max-num-batched-tokens 8192 \\\n --max-num-seqs 16 \\\n --media-io-kwargs '{\"video\": {\"fps\": 2, \"num_frames\": 256}}' \\\n --mm-processor-cache-gb 0 \\\n --prefix-match-unit 16 \\\n --reasoning-parser nemotron_v3 \\\n --speculative_config.max_model_len 1048576 \\\n --speculative_config.method dflash \\\n --speculative_config.model nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4-LiLiCorr \\\n --speculative_config.num_speculative_tokens 7 \\\n --tool-call-parser qwen3_coder \\\n --trust-remote-code \\\n --video-pruning-rate 0.5", "port": 8000, "api_model": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "nvidia/nemotron-3.5-super-vl-120b-a12b/rtx-pro/windows/llama-cpp/q4-k-m/vision-lilicorr", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11669", "backend_build": { "kind": "manual", "revision": "620d2225cc436ab0b001e3200e590e3b286ef9ab" }, "backend_min_version": null, "min_vidmem_gb": null, "precision": "q4-k-m", "variant": "vision-lilicorr", "status": "experimental", "prerequisites": [ "A single NVIDIA RTX PRO 6000 Blackwell Workstation Edition running Windows x86_64.", "Native llama.cpp b11669 CUDA 13.4 with the matching Windows x64 runtime DLLs linked in Notes.", "Both Q4_K_M target shards, their BF16 vision projector, and the linked NVFP4 LiLiCorr draft." ], "limitations": [ "Text-only coding and AA replay were measured with the projector loaded; image/video throughput and output quality were not measured.", "Minimum required memory, exact Windows build, and GPU driver version were not recorded." ], "checkpoint": { "target": "https://huggingface.co/lmstudio-community/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF/blob/main/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-Q4_K_M-00001-of-00002.gguf", "target_label": "lmstudio-community/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-Q4_K_M-00001-of-00002.gguf", "draft": "https://huggingface.co/unsloth/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF/blob/75a5829055f69a0359085ae6bbd0a81b60bc05c6/LiLiCorr/lilicorr-NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4.gguf", "draft_label": "unsloth/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF/LiLiCorr/lilicorr-NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4.gguf" }, "security": null, "topology": null, "runtime": null, "serve": { "label": "Launch LiLiCorr · n=7", "target": "single-node", "instructions": "Run in PowerShell from the directory containing the extracted llama-server.exe and CUDA DLLs.", "command": ".\\llama-server.exe -hf \"lmstudio-community/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF\" `\n -hff \"NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-Q4_K_M-00001-of-00002.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF/resolve/main/mmproj-BF16.gguf\" `\n --batch-size 4096 `\n --ubatch-size 2048 `\n --ctx-size 196608 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 7 `\n --temp 1.0 `\n --top-p 0.95 `\n --flash-attn on `\n --jinja `\n --fit off `\n --host 127.0.0.1 `\n --port 9931 `\n -hfrd \"unsloth/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF\" `\n --spec-draft-model \"LiLiCorr/lilicorr-NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4.gguf\" `\n --spec-draft-ngl 999", "port": 9931, "api_model": "lmstudio-community/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF" }, "launch": null, "sampling": "--temp 1.0; --top-p 0.95", "notes": "Download the [llama.cpp b11669 Windows x64 CUDA 13.4 binaries](https://github.com/ggml-org/llama.cpp/releases/download/b11669/llama-b11669-bin-win-cuda-13.4-x64.zip) and [matching CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/b11669/cudart-llama-bin-win-cuda-13.4-x64.zip).\nExtract both ZIP files into the same directory so `llama-server.exe` and the CUDA DLLs are together. Open PowerShell in that directory before running the launch command.", "clients": { "shell": "'{\"model\":\"lmstudio-community/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"lmstudio-community/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "nvidia/nemotron-3.5-super-vl-120b-a12b/rtx-spark/windows/llama-cpp/q4-k-m/vision-lilicorr", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11669", "backend_build": { "kind": "manual", "revision": "620d2225cc436ab0b001e3200e590e3b286ef9ab" }, "backend_min_version": null, "min_vidmem_gb": 79, "precision": "q4-k-m", "variant": "vision-lilicorr", "status": "experimental", "prerequisites": [ "An RTX Spark running Windows ARM64.", "Native llama.cpp b11669 CUDA 13.4 with the matching Windows ARM64 runtime DLLs linked in Notes.", "Both Q4_K_M target shards, their BF16 vision projector, and the linked NVFP4 LiLiCorr draft." ], "limitations": [ "Text-only coding and AA replay were measured with the projector loaded; image/video throughput and output quality were not measured.", "The 79 GiB memory threshold is based on observed peak use; the true minimum, exact Windows build, and GPU driver version were not recorded." ], "checkpoint": { "target": "https://huggingface.co/lmstudio-community/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF/blob/main/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-Q4_K_M-00001-of-00002.gguf", "target_label": "lmstudio-community/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-Q4_K_M-00001-of-00002.gguf", "draft": "https://huggingface.co/unsloth/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF/blob/75a5829055f69a0359085ae6bbd0a81b60bc05c6/LiLiCorr/lilicorr-NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4.gguf", "draft_label": "unsloth/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF/LiLiCorr/lilicorr-NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4.gguf" }, "security": null, "topology": null, "runtime": null, "serve": { "label": "Launch LiLiCorr · n=5", "target": "single-node", "instructions": "Run in PowerShell from the directory containing the extracted llama-server.exe and CUDA DLLs.", "command": ".\\llama-server.exe -hf \"lmstudio-community/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF\" `\n -hff \"NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-Q4_K_M-00001-of-00002.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF/resolve/main/mmproj-BF16.gguf\" `\n --batch-size 4096 `\n --ubatch-size 4096 `\n --ctx-size 262144 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 5 `\n --temp 1.0 `\n --top-p 0.95 `\n --flash-attn on `\n --jinja `\n --fit off `\n --host 127.0.0.1 `\n --port 9931 `\n -hfrd \"unsloth/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF\" `\n --spec-draft-model \"LiLiCorr/lilicorr-NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4.gguf\" `\n --spec-draft-ngl 999", "port": 9931, "api_model": "lmstudio-community/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF" }, "launch": null, "sampling": "--temp 1.0; --top-p 0.95", "notes": "Download the [llama.cpp b11669 Windows ARM64 CUDA 13.4 binaries](https://github.com/ggml-org/llama.cpp/releases/download/b11669/llama-b11669-bin-win-cuda-13.4-arm64.zip) and [matching CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/b11669/cudart-llama-bin-win-cuda-13.4-arm64.zip).\nExtract both ZIP files into the same directory so `llama-server.exe` and the CUDA DLLs are together. Open PowerShell in that directory before running the launch command.", "clients": { "shell": "'{\"model\":\"lmstudio-community/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"lmstudio-community/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "nvidia/nemotron-bolt-e4b", "display_name": "NVIDIA-Nemotron-3.5-Bolt-E4B-4GB", "builder": "nvidia", "family": "Nemotron", "summary": "Q4_K_M target with BF16 DFlash for native Windows llama.cpp.", "upstream": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-E4B-4GB", "modalities": [ "text" ], "recipes": [ { "id": "nvidia/nemotron-bolt-e4b/geforce-rtx/windows/llama-cpp/q4-k-m", "hardware": "geforce-rtx", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11803", "backend_min_version": null, "min_vidmem_gb": 10.17, "precision": "q4-k-m", "variant": null, "status": "experimental", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build with llama-server.exe on PATH.", "Download the linked target and draft GGUF files into the working directory. Hugging Face access may require authentication." ], "limitations": [ "Text generation at concurrency 1." ], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-E4B-4GB/blob/main/Nemotron-3.5-Bolt-E4B-Q4_K_M.gguf", "target_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-E4B-4GB/Nemotron-3.5-Bolt-E4B-Q4_K_M.gguf", "draft": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-E4B-4GB-DFlash/blob/main/Nemotron-3.5-Bolt-E4B-DFlash-BF16.gguf", "draft_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-E4B-4GB-DFlash/Nemotron-3.5-Bolt-E4B-DFlash-BF16.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11803. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11803\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch DFlash · 7 draft tokens", "target": "single-node", "instructions": "Run in PowerShell from the directory containing both GGUF files. Set $env:GGML_CUDA_MMQ_PREC = \"auto\" before launching. Default mmap is retained.", "command": "llama-server.exe `\n -m ./Nemotron-3.5-Bolt-E4B-Q4_K_M.gguf `\n --no-mmproj `\n --host 127.0.0.1 `\n --port 8080 `\n --batch-size 4096 `\n --ubatch-size 1024 `\n --ctx-size 98304 `\n --parallel 1 `\n -ngl 999 `\n --n-predict 8192 `\n -fit off `\n --spec-type draft-dflash `\n --spec-draft-n-max 7 `\n --spec-draft-model ./Nemotron-3.5-Bolt-E4B-DFlash-BF16.gguf `\n --spec-draft-backend-sampling `\n --backend-sampling `\n --temp 0 `\n --top-p 1 `\n --top-k 1 `\n --min-p 0 `\n --presence-penalty 0 `\n --repeat-penalty 1 `\n -fa on `\n -ctk f16 `\n -ctv f16 `\n --jinja `\n --seed 1234 `\n --threads 12 `\n --threads-batch 12 `\n --metrics `\n --no-webui `\n --chat-template-kwargs '{\"enable_thinking\":true,\"preserve_thinking\":true}' `\n --spec-draft-p-min 0 `\n -ngld 999 `\n --spec-draft-type-k f16 `\n --spec-draft-type-v f16 `\n --alias bolt-q4-k-m", "port": 8080, "api_model": "bolt-q4-k-m" }, "launch": null, "sampling": "Temperature 0; top-p 1; top-k 1; min-p 0; seed 1234.", "notes": null, "clients": { "shell": "'{\"model\":\"bolt-q4-k-m\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:8080/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"bolt-q4-k-m\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "nvidia/nemotron-bolt-e4b/rtx-pro/windows/llama-cpp/q4-k-m", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11803", "backend_min_version": null, "min_vidmem_gb": 10.1, "precision": "q4-k-m", "variant": null, "status": "experimental", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build with llama-server.exe on PATH.", "Download the linked target and draft GGUF files into the working directory. Hugging Face access may require authentication." ], "limitations": [ "Text generation at concurrency 1." ], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-E4B-4GB/blob/main/Nemotron-3.5-Bolt-E4B-Q4_K_M.gguf", "target_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-E4B-4GB/Nemotron-3.5-Bolt-E4B-Q4_K_M.gguf", "draft": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-E4B-4GB-DFlash/blob/main/Nemotron-3.5-Bolt-E4B-DFlash-BF16.gguf", "draft_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-E4B-4GB-DFlash/Nemotron-3.5-Bolt-E4B-DFlash-BF16.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11803. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11803\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch DFlash · 7 draft tokens", "target": "single-node", "instructions": "Run in PowerShell from the directory containing both GGUF files. Set $env:GGML_CUDA_MMQ_PREC = \"auto\" before launching. Default mmap is retained.", "command": "llama-server.exe `\n -m ./Nemotron-3.5-Bolt-E4B-Q4_K_M.gguf `\n --no-mmproj `\n --host 127.0.0.1 `\n --port 8080 `\n --batch-size 2048 `\n --ubatch-size 1024 `\n --ctx-size 98304 `\n --parallel 1 `\n -ngl 999 `\n --n-predict 8192 `\n -fit off `\n --spec-type draft-dflash `\n --spec-draft-n-max 7 `\n --spec-draft-model ./Nemotron-3.5-Bolt-E4B-DFlash-BF16.gguf `\n --spec-draft-backend-sampling `\n --backend-sampling `\n --temp 0 `\n --top-p 1 `\n --top-k 1 `\n --min-p 0 `\n --presence-penalty 0 `\n --repeat-penalty 1 `\n -fa on `\n -ctk f16 `\n -ctv f16 `\n --jinja `\n --seed 1234 `\n --threads 12 `\n --threads-batch 12 `\n --metrics `\n --no-webui `\n --chat-template-kwargs '{\"enable_thinking\":true,\"preserve_thinking\":true}' `\n --spec-draft-p-min 0 `\n -ngld 999 `\n --spec-draft-type-k f16 `\n --spec-draft-type-v f16 `\n --alias bolt-q4-k-m", "port": 8080, "api_model": "bolt-q4-k-m" }, "launch": null, "sampling": "Temperature 0; top-p 1; top-k 1; min-p 0; seed 1234.", "notes": null, "clients": { "shell": "'{\"model\":\"bolt-q4-k-m\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:8080/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"bolt-q4-k-m\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "nvidia/nemotron-bolt-e4b/rtx-spark/windows/llama-cpp/q4-k-m", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11803", "backend_min_version": null, "min_vidmem_gb": 14.55, "precision": "q4-k-m", "variant": null, "status": "experimental", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build with llama-server.exe on PATH.", "Download the linked target and draft GGUF files into the working directory. Hugging Face access may require authentication." ], "limitations": [ "Text generation at concurrency 1." ], "checkpoint": { "target": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-E4B-4GB/blob/main/Nemotron-3.5-Bolt-E4B-Q4_K_M.gguf", "target_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-E4B-4GB/Nemotron-3.5-Bolt-E4B-Q4_K_M.gguf", "draft": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-E4B-4GB-DFlash/blob/main/Nemotron-3.5-Bolt-E4B-DFlash-BF16.gguf", "draft_label": "nvidia/NVIDIA-Nemotron-3.5-Bolt-E4B-4GB-DFlash/Nemotron-3.5-Bolt-E4B-DFlash-BF16.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11803. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11803\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch DFlash · 7 draft tokens", "target": "single-node", "instructions": "Run in PowerShell from the directory containing both GGUF files. Set $env:GGML_CUDA_MMQ_PREC = \"auto\" before launching. Default mmap is retained.", "command": "llama-server.exe `\n -m ./Nemotron-3.5-Bolt-E4B-Q4_K_M.gguf `\n --no-mmproj `\n --host 127.0.0.1 `\n --port 8080 `\n --batch-size 4096 `\n --ubatch-size 512 `\n --ctx-size 98304 `\n --parallel 1 `\n -ngl 999 `\n --n-predict 8192 `\n -fit off `\n --spec-type draft-dflash `\n --spec-draft-n-max 7 `\n --spec-draft-model ./Nemotron-3.5-Bolt-E4B-DFlash-BF16.gguf `\n --spec-draft-backend-sampling `\n --temp 0 `\n --top-p 1 `\n --top-k 1 `\n --min-p 0 `\n --presence-penalty 0 `\n --repeat-penalty 1 `\n -fa on `\n -ctk f16 `\n -ctv f16 `\n --jinja `\n --seed 1234 `\n --threads 12 `\n --threads-batch 12 `\n --metrics `\n --no-webui `\n --chat-template-kwargs '{\"enable_thinking\":true,\"preserve_thinking\":true}' `\n --spec-draft-p-min 0 `\n -ngld 999 `\n --spec-draft-type-k f16 `\n --spec-draft-type-v f16 `\n --alias bolt-q4-k-m", "port": 8080, "api_model": "bolt-q4-k-m" }, "launch": null, "sampling": "Temperature 0; top-p 1; top-k 1; min-p 0; seed 1234.", "notes": null, "clients": { "shell": "'{\"model\":\"bolt-q4-k-m\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:8080/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8080/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"bolt-q4-k-m\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "qwen/qwen3.5-122b-a10b", "display_name": "Qwen3.5-122B-A10B", "builder": "qwen", "family": "Qwen3.5", "summary": "Qwen 3.5 (122B-A10B) text-generation model.", "upstream": null, "modalities": [ "text" ], "recipes": [ { "id": "qwen/qwen3.5-122b-a10b/dgx-spark/linux/vllm/nvfp4/mtp", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.31.0", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "nvfp4", "variant": "mtp", "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.5-122B-A10B-NVFP4", "target_label": "nvidia/Qwen3.5-122B-A10B-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.31.0", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.31.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.31.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Qwen3.5-122B-A10B-NVFP4 \\\n --max-model-len 262144 \\\n --gpu-memory-utilization 0.8 \\\n --max-num-seqs 8 \\\n --port 8000 \\\n --host 0.0.0.0 \\\n --load-format safetensors \\\n --enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_xml \\\n --reasoning-parser qwen3 \\\n --max-num-batched-tokens 8192 \\\n --attention-backend flashinfer \\\n --kv-cache-dtype fp8 \\\n --moe-backend marlin \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}'", "port": 8000, "api_model": "nvidia/Qwen3.5-122B-A10B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.5-122B-A10B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.5-122B-A10B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.5-122b-a10b/rtx-pro/windows/llama-cpp/q4-k-m", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": null, "backend_min_version": "b10505", "min_vidmem_gb": 96, "precision": "q4-k-m", "variant": null, "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.5-122B-A10B-MTP-GGUF/tree/main/UD-Q4_K_M", "target_label": "unsloth/Qwen3.5-122B-A10B-MTP-GGUF", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Downloads the latest release. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/latest\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch", "target": "single-node", "instructions": "Run in PowerShell.", "command": "llama-server -m UD-Q4_K_M/Qwen3.5-122B-A10B-UD-Q4_K_M-00001-of-00003.gguf `\n -b 4096 `\n -ub 4096 `\n --ctx-size 262144 `\n --spec-type draft-mtp `\n --temp 1.0 `\n --top-k 20 `\n --top-p 0.95 `\n --min-p 0 `\n --repeat-penalty 1 `\n --presence-penalty 0 `\n -np 1", "port": null, "api_model": "UD-Q4_K_M/Qwen3.5-122B-A10B-UD-Q4_K_M-00001-of-00003.gguf" }, "launch": null, "sampling": "Temp=1.0; Top K=20; Top P=0.95; Min P=0; Repetition Penalty=1; Presence Penalty=0", "notes": "Download all three GGUF shards under `UD-Q4_K_M`; `-m` points to the first shard.", "clients": null } ] }, { "id": "qwen/qwen3.5-9b", "display_name": "Qwen3.5-9B", "builder": "qwen", "family": "Qwen3.5", "summary": "Qwen 3.5 (9B) text-generation model.", "upstream": null, "modalities": [ "text", "image" ], "recipes": [ { "id": "qwen/qwen3.5-9b/geforce-rtx/windows/llama-cpp/18gb/q4-k-xl/vision-mtp", "hardware": "geforce-rtx", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11256", "min_vidmem_gb": 18, "precision": "q4-k-xl", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint and projector." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.5-9B-MTP-GGUF/blob/main/Qwen3.5-9B-UD-Q4_K_XL.gguf", "target_label": "unsloth/Qwen3.5-9B-MTP-GGUF/Qwen3.5-9B-UD-Q4_K_XL.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and projector GGUFs from Hugging Face.", "command": "llama-server.exe -hf \"unsloth/Qwen3.5-9B-MTP-GGUF\" `\n -hff \"Qwen3.5-9B-UD-Q4_K_XL.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.5-9B-MTP-GGUF/resolve/9716a636ee4bddc3fed678220b7a33dd2a4160ae/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 512 `\n --ctx-size 262144 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 3 `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.5-9B-MTP-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.5-9B-MTP-GGUF/blob/9716a636ee4bddc3fed678220b7a33dd2a4160ae/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.5-9B-MTP-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.5-9B-MTP-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.5-9b/rtx-pro/windows/llama-cpp/18gb/q4-k-xl/vision-mtp", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11256", "min_vidmem_gb": 18, "precision": "q4-k-xl", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.5-9B-MTP-GGUF/blob/main/Qwen3.5-9B-UD-Q4_K_XL.gguf", "target_label": "unsloth/Qwen3.5-9B-MTP-GGUF/Qwen3.5-9B-UD-Q4_K_XL.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named checkpoint files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/Qwen3.5-9B-MTP-GGUF\" `\n -hff \"Qwen3.5-9B-UD-Q4_K_XL.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.5-9B-MTP-GGUF/resolve/9716a636ee4bddc3fed678220b7a33dd2a4160ae/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 512 `\n --ctx-size 262144 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 3 `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-k 20 `\n --top-p 0.95 `\n --min-p 0 `\n --repeat-penalty 1 `\n --presence-penalty 0", "port": 9931, "api_model": "unsloth/Qwen3.5-9B-MTP-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-k=20; --top-p=0.95; --min-p=0; --repeat-penalty=1; --presence-penalty=0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.5-9B-MTP-GGUF/blob/9716a636ee4bddc3fed678220b7a33dd2a4160ae/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.5-9B-MTP-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.5-9B-MTP-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.5-9b/rtx-spark/windows/llama-cpp/17gb/q4-k-xl/vision-mtp", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11572", "backend_min_version": "b11229", "min_vidmem_gb": 17, "precision": "q4-k-xl", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build b11229, source commit c2a9e1606807970f4ee3167bacd951699c89caea, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.5-9B-MTP-GGUF/blob/main/Qwen3.5-9B-UD-Q4_K_XL.gguf", "target_label": "unsloth/Qwen3.5-9B-MTP-GGUF/Qwen3.5-9B-UD-Q4_K_XL.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11572. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11572\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and draft files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/Qwen3.5-9B-MTP-GGUF\" `\n -hff \"Qwen3.5-9B-UD-Q4_K_XL.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.5-9B-MTP-GGUF/resolve/9716a636ee4bddc3fed678220b7a33dd2a4160ae/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 512 `\n --ctx-size 262144 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 3 `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-k 20 `\n --top-p 0.95 `\n --min-p 0 `\n --repeat-penalty 1 `\n --presence-penalty 0", "port": 9931, "api_model": "unsloth/Qwen3.5-9B-MTP-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-k=20; --top-p=0.95; --min-p=0; --repeat-penalty=1; --presence-penalty=0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.5-9B-MTP-GGUF/blob/9716a636ee4bddc3fed678220b7a33dd2a4160ae/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.5-9B-MTP-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.5-9B-MTP-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "qwen/qwen3.6-27b", "display_name": "Qwen3.6-27B", "builder": "qwen", "family": "Qwen3.6", "summary": "Qwen 3.6 (27B) text-generation model.", "upstream": null, "modalities": [ "text" ], "recipes": [ { "id": "qwen/qwen3.6-27b/dgx-spark-2gpu/linux/vllm/nvfp4-w4a16/mtp", "hardware": "dgx-spark-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.31.0", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "nvfp4-w4a16", "variant": "mtp", "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.6-27B-NVFP4", "target_label": "nvidia/Qwen3.6-27B-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× DGX Spark; one node rank per Spark; tensor parallel 2", "gpu_count": 2, "node_count": 2, "tensor_parallel_size": 2 }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.31.0", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.31.0", "setup_label": "Start container", "setup_target": "all-nodes", "setup_instructions": "Run on every node after setting `HF_TOKEN`; replace the node-specific placeholders.", "setup_command": "# Example two-Spark values (adjust to your network):\n# HEAD_IP=\"10.100.32.1\" # head node IP on the ETH subnet\n# ETH_IF=\"enp1s0f0np0\" # ConnectX baseboard ethernet (socket/control)\n# IB_IF=\"rocep1s0f0,roceP2p1s0f0\" # RoCE HCA names (data plane) - use both \"twins\"!\nexport HEAD_IP=\"\"\nexport ETH_IF=\"\"\nexport IB_IF=\",\"\nexport MASTER_PORT=29501\nexport NODE_RANK=\"<0-1>\"\nexport IMAGE=\"vllm/vllm-openai:v0.31.0\"\nexport HF_CACHE_DIR=\"$HOME/.cache/huggingface\"\n\nmkdir -p \"$HF_CACHE_DIR\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\"\n\ndocker run --rm -it \\\n --privileged \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit nofile=1048576:1048576 \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e VLLM_FLOAT32_MATMUL_PRECISION=high \\\n -e NODE_RANK=\"$NODE_RANK\" \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HF_CACHE_DIR:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n --entrypoint /bin/bash \\\n \"$IMAGE\"" }, "serve": { "label": "Launch head", "target": "head", "instructions": null, "command": "export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\n\nvllm serve nvidia/Qwen3.6-27B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --attention-backend flashinfer \\\n --gpu-memory-utilization 0.5 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":5,\"draft_tensor_parallel_size\": 2}' \\\n --load-format fastsafetensors \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice \\\n --tensor-parallel-size 2 \\\n --nnodes 2 \\\n --node-rank 0 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\"", "port": 8000, "api_model": "nvidia/Qwen3.6-27B-NVFP4", "additional_commands": [ { "id": "worker", "label": "Launch worker", "target": "workers", "instructions": null, "command": "export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\n\nvllm serve nvidia/Qwen3.6-27B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --attention-backend flashinfer \\\n --gpu-memory-utilization 0.5 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":5,\"draft_tensor_parallel_size\": 2}' \\\n --load-format fastsafetensors \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice \\\n --tensor-parallel-size 2 \\\n --nnodes 2 \\\n --node-rank 1 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\" \\\n --headless" } ] }, "launch": null, "sampling": null, "notes": "Use the same CX7 port on every node; assign both port subinterfaces on separate subnets; cache the image and checkpoint on every node; start workers before the head. Docker runs privileged.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.6-27B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.6-27B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.6-27b/dgx-spark/linux/vllm/nvfp4-w4a16/mtp", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.31.0", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "nvfp4-w4a16", "variant": "mtp", "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.6-27B-NVFP4", "target_label": "nvidia/Qwen3.6-27B-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.31.0", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.31.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.31.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Qwen3.6-27B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --attention-backend flashinfer \\\n --gpu-memory-utilization 0.5 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":5}' \\\n --load-format fastsafetensors \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice", "port": 8000, "api_model": "nvidia/Qwen3.6-27B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.6-27B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.6-27B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.6-27b/dgx-station/linux/vllm/nvfp4-w4a16/mtp", "hardware": "dgx-station", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.31.0", "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4-w4a16", "variant": "mtp", "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.6-27B-NVFP4", "target_label": "nvidia/Qwen3.6-27B-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Station", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.31.0", "image_url": "https://hub.docker.com/layers/vllm/vllm-openai/v0.31.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.31.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Qwen3.6-27B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --attention-backend flashinfer \\\n --gpu-memory-utilization 0.92 \\\n --max-model-len 262144 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}' \\\n --load-format fastsafetensors \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice", "port": 8000, "api_model": "nvidia/Qwen3.6-27B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.6-27B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.6-27B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.6-27b/geforce-rtx/linux/vllm/nvfp4-w4a16/mtp", "hardware": "geforce-rtx", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.31.0", "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4-w4a16", "variant": "mtp", "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.6-27B-NVFP4", "target_label": "nvidia/Qwen3.6-27B-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× GeForce RTX GPUs", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.31.0", "image_url": "https://hub.docker.com/layers/vllm/vllm-openai/v0.31.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.31.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "export VLLM_HAS_FLASHINFER_CUBIN=1\n\nvllm serve nvidia/Qwen3.6-27B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --tensor-parallel-size 1 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --attention-backend FLASHINFER \\\n --attention-config '{\"use_trtllm_attention\": true}' \\\n --gpu-memory-utilization 0.93 \\\n --max-model-len 36608 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 4096 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --async-scheduling \\\n --skip-mm-profiling \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}' \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice", "port": 8000, "api_model": "nvidia/Qwen3.6-27B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.6-27B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.6-27B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.6-27b/rtx-pro/linux/vllm/nvfp4-w4a16/mtp", "hardware": "rtx-pro", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.31.0", "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4-w4a16", "variant": "mtp", "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.6-27B-NVFP4", "target_label": "nvidia/Qwen3.6-27B-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× RTX PRO 6000", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.31.0", "image_url": "https://hub.docker.com/layers/vllm/vllm-openai/v0.31.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.31.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Qwen3.6-27B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --attention-backend flashinfer \\\n --gpu-memory-utilization 0.9 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}' \\\n --load-format fastsafetensors \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice", "port": 8000, "api_model": "nvidia/Qwen3.6-27B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.6-27B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.6-27B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "qwen/qwen3.6-35b-a3b", "display_name": "Qwen3.6-35B-A3B", "builder": "qwen", "family": "Qwen3.6", "summary": "Qwen 3.6 (35B-A3B) text-generation model.", "upstream": null, "modalities": [ "text", "image" ], "recipes": [ { "id": "qwen/qwen3.6-35b-a3b/dgx-spark/linux/vllm/128gb/nvfp4-w4a16/mtp", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.31.0+88641915.nv26.10.73893335", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "nvfp4-w4a16", "variant": "mtp", "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.6-35B-A3B-NVFP4", "target_label": "nvidia/Qwen3.6-35B-A3B-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container", "image": "nvcr.io/nvidia/vllm:26.10.pre1-py3", "image_url": "https://catalog.ngc.nvidia.com/orgs/nvidia/containers/vllm?version=26.10.pre1-py3", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment. The image is public on NGC, so no registry login is needed; if `docker pull` fails with `unauthorized`, clear a stale login with `docker logout nvcr.io`.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n nvcr.io/nvidia/vllm:26.10.pre1-py3" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Qwen3.6-35B-A3B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --tensor-parallel-size 1 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --attention-backend flashinfer \\\n --moe-backend marlin \\\n --gpu-memory-utilization 0.8 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"moe_backend\":\"triton\"}' \\\n --load-format safetensors \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice", "port": 8000, "api_model": "nvidia/Qwen3.6-35B-A3B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.6-35B-A3B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.6-35B-A3B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.6-35b-a3b/dgx-spark/linux/vllm/64gb/nvfp4-w4a16/mtp", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.28.0", "backend_min_version": null, "min_vidmem_gb": 64, "precision": "nvfp4-w4a16", "variant": "mtp", "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.6-35B-A3B-NVFP4", "target_label": "nvidia/Qwen3.6-35B-A3B-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.28.0", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.28.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.28.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Qwen3.6-35B-A3B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --tensor-parallel-size 1 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --attention-backend flashinfer \\\n --moe-backend marlin \\\n --gpu-memory-utilization 0.8 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}' \\\n --load-format safetensors \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice", "port": 8000, "api_model": "nvidia/Qwen3.6-35B-A3B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.6-35B-A3B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.6-35B-A3B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.6-35b-a3b/geforce-rtx/linux/llama-cpp/q4-k-m/mtp", "hardware": "geforce-rtx", "platform": { "os": "linux", "execution": "native" }, "backend": "llama-cpp", "backend_version": null, "backend_min_version": "b10505", "min_vidmem_gb": 26, "precision": "q4-k-m", "variant": "mtp", "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/main/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", "target_label": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": null, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in a Linux shell. llama-server downloads and caches the target GGUF from Hugging Face.", "command": "llama-server -hf \"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\" \\\n -hff \"Qwen3.6-35B-A3B-UD-Q4_K_M.gguf\" \\\n --no-mmproj \\\n --backend-sampling \\\n --batch-size 2048 \\\n --ubatch-size 2048 \\\n --ctx-size 65536 \\\n --parallel 1 \\\n --spec-type draft-mtp \\\n --spec-draft-n-max 2 \\\n --temp 1.0 \\\n --top-p 0.95 \\\n --top-k 20 \\\n --min-p 0.0 \\\n --presence-penalty 0.0 \\\n --repeat-penalty 1.0", "port": null, "api_model": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF" }, "launch": null, "sampling": "Target backend sampling: ON; speculative draft backend sampling: default; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "The public [Q4_K_M target GGUF](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/main/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf) contains integrated MTP layers, so no separate draft checkpoint is required. Use MTP depth 2.", "clients": null }, { "id": "qwen/qwen3.6-35b-a3b/geforce-rtx/linux/vllm/nvfp4-w4a16/mtp", "hardware": "geforce-rtx", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": null, "backend_min_version": null, "min_vidmem_gb": 32, "precision": "nvfp4-w4a16", "variant": "mtp", "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.6-35B-A3B-NVFP4", "target_label": "nvidia/Qwen3.6-35B-A3B-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× GeForce RTX GPUs", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.31.0", "image_url": "https://hub.docker.com/layers/vllm/vllm-openai/v0.31.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.31.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "export VLLM_HAS_FLASHINFER_CUBIN=1\n\nvllm serve nvidia/Qwen3.6-35B-A3B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --tensor-parallel-size 1 \\\n --trust-remote-code \\\n --quantization modelopt_fp4 \\\n --kv-cache-dtype fp8 \\\n --block-size 128 \\\n --moe-backend marlin \\\n --attention-backend FLASHINFER \\\n --attention-config '{\"use_trtllm_attention\": true}' \\\n --gpu-memory-utilization 0.85 \\\n --max-model-len 65536 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --async-scheduling \\\n --skip-mm-profiling \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"moe_backend\":\"flashinfer_cutlass\"}' \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice", "port": 8000, "api_model": "nvidia/Qwen3.6-35B-A3B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.6-35B-A3B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.6-35B-A3B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.6-35b-a3b/geforce-rtx/windows/llama-cpp/24gb/iq4-xs/vision-mtp", "hardware": "geforce-rtx", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11256", "min_vidmem_gb": 24, "precision": "iq4-xs", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint and projector." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/main/Qwen3.6-35B-A3B-UD-IQ4_XS.gguf", "target_label": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF/Qwen3.6-35B-A3B-UD-IQ4_XS.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and projector GGUFs from Hugging Face.", "command": "llama-server.exe -hf \"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\" `\n -hff \"Qwen3.6-35B-A3B-UD-IQ4_XS.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/resolve/5bc3e238d916f48a861bac2f8a1990a0e9b7e98d/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 1024 `\n --ctx-size 164864 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 2 `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/5bc3e238d916f48a861bac2f8a1990a0e9b7e98d/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.6-35b-a3b/geforce-rtx/windows/llama-cpp/30gb/q4-k-m/vision-mtp", "hardware": "geforce-rtx", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11256", "min_vidmem_gb": 30, "precision": "q4-k-m", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint and projector." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/main/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", "target_label": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and projector GGUFs from Hugging Face.", "command": "llama-server.exe -hf \"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\" `\n -hff \"Qwen3.6-35B-A3B-UD-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/resolve/5bc3e238d916f48a861bac2f8a1990a0e9b7e98d/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 2048 `\n --ctx-size 216064 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 2 `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/5bc3e238d916f48a861bac2f8a1990a0e9b7e98d/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.6-35b-a3b/rtx-pro/linux/llama-cpp/q4-k-m/mtp", "hardware": "rtx-pro", "platform": { "os": "linux", "execution": "native" }, "backend": "llama-cpp", "backend_version": null, "backend_min_version": "b10505", "min_vidmem_gb": 26, "precision": "q4-k-m", "variant": "mtp", "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/main/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", "target_label": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": null, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in a Linux shell. llama-server downloads and caches the target GGUF from Hugging Face.", "command": "llama-server -hf \"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\" \\\n -hff \"Qwen3.6-35B-A3B-UD-Q4_K_M.gguf\" \\\n --no-mmproj \\\n --backend-sampling \\\n --batch-size 2048 \\\n --ubatch-size 2048 \\\n --ctx-size 65536 \\\n --parallel 1 \\\n --spec-type draft-mtp \\\n --spec-draft-n-max 2 \\\n --temp 1.0 \\\n --top-p 0.95 \\\n --top-k 20 \\\n --min-p 0.0 \\\n --presence-penalty 0.0 \\\n --repeat-penalty 1.0", "port": null, "api_model": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF" }, "launch": null, "sampling": "Target backend sampling: ON; speculative draft backend sampling: default; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "The public [Q4_K_M target GGUF](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/main/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf) contains integrated MTP layers, so no separate draft checkpoint is required. Use MTP depth 2.", "clients": null }, { "id": "qwen/qwen3.6-35b-a3b/rtx-pro/linux/vllm/nvfp4-w4a16/mtp", "hardware": "rtx-pro", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": null, "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4-w4a16", "variant": "mtp", "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.6-35B-A3B-NVFP4", "target_label": "nvidia/Qwen3.6-35B-A3B-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× RTX PRO 6000", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.31.0", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.31.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.31.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "export VLLM_HAS_FLASHINFER_CUBIN=1\n\nvllm serve nvidia/Qwen3.6-35B-A3B-NVFP4 \\\n --trust-remote-code \\\n --quantization modelopt_fp4 \\\n --kv-cache-dtype fp8 \\\n --block-size 128 \\\n --moe-backend marlin \\\n --attention-backend FLASHINFER \\\n --attention-config '{\"use_trtllm_attention\":true}' \\\n --gpu-memory-utilization 0.90 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --async-scheduling \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --skip-mm-profiling \\\n --load-format fastsafetensors \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"moe_backend\":\"flashinfer_cutlass\"}'", "port": null, "api_model": "nvidia/Qwen3.6-35B-A3B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": null }, { "id": "qwen/qwen3.6-35b-a3b/rtx-pro/windows/llama-cpp/24gb/iq4-xs/vision-mtp", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11256", "min_vidmem_gb": 24, "precision": "iq4-xs", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/main/Qwen3.6-35B-A3B-UD-IQ4_XS.gguf", "target_label": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF/Qwen3.6-35B-A3B-UD-IQ4_XS.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named checkpoint files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\" `\n -hff \"Qwen3.6-35B-A3B-UD-IQ4_XS.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/resolve/5bc3e238d916f48a861bac2f8a1990a0e9b7e98d/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 512 `\n --ctx-size 163840 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 2 `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/5bc3e238d916f48a861bac2f8a1990a0e9b7e98d/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.6-35b-a3b/rtx-pro/windows/llama-cpp/32gb/q4-k-m/vision-mtp", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11256", "min_vidmem_gb": 32, "precision": "q4-k-m", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/main/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", "target_label": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named checkpoint files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\" `\n -hff \"Qwen3.6-35B-A3B-UD-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/resolve/5bc3e238d916f48a861bac2f8a1990a0e9b7e98d/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 2048 `\n --ctx-size 262144 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 2 `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/5bc3e238d916f48a861bac2f8a1990a0e9b7e98d/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.6-35b-a3b/rtx-spark/windows/llama-cpp/24gb/iq4-xs/vision-mtp", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11572", "backend_min_version": "b11236", "min_vidmem_gb": 24, "precision": "iq4-xs", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build b11236, source commit f1ea206218210afb913ae2f5d2c51faed35915da, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/main/Qwen3.6-35B-A3B-UD-IQ4_XS.gguf", "target_label": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF/Qwen3.6-35B-A3B-UD-IQ4_XS.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11572. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11572\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and draft files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\" `\n -hff \"Qwen3.6-35B-A3B-UD-IQ4_XS.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/resolve/5bc3e238d916f48a861bac2f8a1990a0e9b7e98d/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 4096 `\n --ctx-size 101376 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 2 `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/5bc3e238d916f48a861bac2f8a1990a0e9b7e98d/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.6-35b-a3b/rtx-spark/windows/llama-cpp/28gb/q4-k-m/vision-mtp", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11572", "backend_min_version": "b11236", "min_vidmem_gb": 28, "precision": "q4-k-m", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build b11236, source commit f1ea206218210afb913ae2f5d2c51faed35915da, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/main/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", "target_label": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11572. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11572\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and draft files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\" `\n -hff \"Qwen3.6-35B-A3B-UD-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/resolve/5bc3e238d916f48a861bac2f8a1990a0e9b7e98d/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 4096 `\n --ctx-size 99328 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 2 `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/5bc3e238d916f48a861bac2f8a1990a0e9b7e98d/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.6-35b-a3b/rtx-spark/windows/llama-cpp/34gb/q4-k-m/vision-mtp", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11572", "backend_min_version": "b11236", "min_vidmem_gb": 34, "precision": "q4-k-m", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build b11236, source commit f1ea206218210afb913ae2f5d2c51faed35915da, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/main/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", "target_label": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11572. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11572\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and draft files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\" `\n -hff \"Qwen3.6-35B-A3B-UD-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/resolve/5bc3e238d916f48a861bac2f8a1990a0e9b7e98d/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 4096 `\n --ctx-size 262144 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 2 `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/blob/5bc3e238d916f48a861bac2f8a1990a0e9b7e98d/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.6-35B-A3B-MTP-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "qwen/qwen3.8-27b", "display_name": "Qwen3.8-27B", "builder": "qwen", "family": "Qwen3.8", "summary": "Qwen 3.8 27B text-generation model.", "upstream": null, "modalities": [ "text", "image" ], "recipes": [ { "id": "qwen/qwen3.8-27b/dgx-spark-2gpu/linux/vllm/128gb/nvfp4", "hardware": "dgx-spark-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.31.0", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "nvfp4", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.8-27B-NVFP4", "target_label": "nvidia/Qwen3.8-27B-NVFP4", "draft": "https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2", "draft_label": "z-lab/Qwen3.8-27B-DFlash2" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× DGX Spark; one node rank per Spark; tensor parallel 2", "gpu_count": 2, "node_count": 2, "tensor_parallel_size": 2 }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.31.0", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.31.0", "setup_label": "Start container", "setup_target": "all-nodes", "setup_instructions": "Run on every node after setting `HF_TOKEN`; replace the node-specific placeholders.", "setup_command": "# Example two-Spark values (adjust to your network):\n# HEAD_IP=\"10.100.32.1\" # head node IP on the ETH subnet\n# ETH_IF=\"enp1s0f0np0\" # ConnectX baseboard ethernet (socket/control)\n# IB_IF=\"rocep1s0f0,roceP2p1s0f0\" # RoCE HCA names (data plane) - use both \"twins\"!\nexport HEAD_IP=\"\"\nexport ETH_IF=\"\"\nexport IB_IF=\",\"\nexport MASTER_PORT=29501\nexport NODE_RANK=\"<0-1>\"\nexport IMAGE=\"vllm/vllm-openai:v0.31.0\"\nexport HF_CACHE_DIR=\"$HOME/.cache/huggingface\"\n\nmkdir -p \"$HF_CACHE_DIR\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\"\n\ndocker run --rm -it \\\n --privileged \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit nofile=1048576:1048576 \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e VLLM_FLOAT32_MATMUL_PRECISION=high \\\n -e NODE_RANK=\"$NODE_RANK\" \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HF_CACHE_DIR:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n --entrypoint /bin/bash \\\n \"$IMAGE\"" }, "serve": { "label": "Launch head", "target": "head", "instructions": null, "command": "export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\n\nvllm serve nvidia/Qwen3.8-27B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --gpu-memory-utilization 0.8 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 16384 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"dflash\",\"model\": \"z-lab/Qwen3.8-27B-DFlash2\", \"num_speculative_tokens\":8, \"draft_tensor_parallel_size\": 2}' \\\n --load-format safetensors \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice \\\n --tensor-parallel-size 2 \\\n --nnodes 2 \\\n --node-rank $NODE_RANK \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\"", "port": 8000, "api_model": "nvidia/Qwen3.8-27B-NVFP4", "additional_commands": [ { "id": "worker", "label": "Launch worker", "target": "workers", "instructions": null, "command": "export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\n\nvllm serve nvidia/Qwen3.8-27B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --gpu-memory-utilization 0.8 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 16384 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"dflash\",\"model\": \"z-lab/Qwen3.8-27B-DFlash2\", \"num_speculative_tokens\":8, \"draft_tensor_parallel_size\": 2}' \\\n --load-format safetensors \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice \\\n --tensor-parallel-size 2 \\\n --nnodes 2 \\\n --node-rank $NODE_RANK \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\" \\\n --headless" } ] }, "launch": null, "sampling": null, "notes": "Use the same CX7 port on every node; assign both port subinterfaces on separate subnets; cache the image and checkpoint on every node; start workers before the head. Docker runs privileged.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.8-27B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.8-27B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-27b/dgx-spark-2gpu/linux/vllm/64gb/nvfp4", "hardware": "dgx-spark-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.28.0", "backend_min_version": null, "min_vidmem_gb": 64, "precision": "nvfp4", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.8-27B-NVFP4", "target_label": "nvidia/Qwen3.8-27B-NVFP4", "draft": "https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2", "draft_label": "z-lab/Qwen3.8-27B-DFlash2" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× DGX Spark; one node rank per Spark; tensor parallel 2", "gpu_count": 2, "node_count": 2, "tensor_parallel_size": 2 }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.28.0", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.28.0", "setup_label": "Start container", "setup_target": "all-nodes", "setup_instructions": "Run on every node after setting `HF_TOKEN`; replace the node-specific placeholders.", "setup_command": "# Example two-Spark values (adjust to your network):\n# HEAD_IP=\"10.100.32.1\" # head node IP on the ETH subnet\n# ETH_IF=\"enp1s0f0np0\" # ConnectX baseboard ethernet (socket/control)\n# IB_IF=\"rocep1s0f0,roceP2p1s0f0\" # RoCE HCA names (data plane) - use both \"twins\"!\nexport HEAD_IP=\"\"\nexport ETH_IF=\"\"\nexport IB_IF=\",\"\nexport MASTER_PORT=29501\nexport NODE_RANK=\"<0-1>\"\nexport IMAGE=\"vllm/vllm-openai:v0.28.0\"\nexport HF_CACHE_DIR=\"$HOME/.cache/huggingface\"\n\nmkdir -p \"$HF_CACHE_DIR\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\"\n\ndocker run --rm -it \\\n --privileged \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit nofile=1048576:1048576 \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e VLLM_FLOAT32_MATMUL_PRECISION=high \\\n -e NODE_RANK=\"$NODE_RANK\" \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HF_CACHE_DIR:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n --entrypoint /bin/bash \\\n \"$IMAGE\"" }, "serve": { "label": "Launch head", "target": "head", "instructions": null, "command": "export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\n\nvllm serve nvidia/Qwen3.8-27B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --gpu-memory-utilization 0.8 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 16384 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"dflash\",\"model\": \"z-lab/Qwen3.8-27B-DFlash2\", \"num_speculative_tokens\":8, \"draft_tensor_parallel_size\": 2}' \\\n --load-format safetensors \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice \\\n --tensor-parallel-size 2 \\\n --nnodes 2 \\\n --node-rank $NODE_RANK \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\"", "port": 8000, "api_model": "nvidia/Qwen3.8-27B-NVFP4", "additional_commands": [ { "id": "worker", "label": "Launch worker", "target": "workers", "instructions": null, "command": "export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\n\nvllm serve nvidia/Qwen3.8-27B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --gpu-memory-utilization 0.8 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 16384 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"dflash\",\"model\": \"z-lab/Qwen3.8-27B-DFlash2\", \"num_speculative_tokens\":8, \"draft_tensor_parallel_size\": 2}' \\\n --load-format safetensors \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice \\\n --tensor-parallel-size 2 \\\n --nnodes 2 \\\n --node-rank $NODE_RANK \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\" \\\n --headless" } ] }, "launch": null, "sampling": null, "notes": "Use the same CX7 port on every node; assign both port subinterfaces on separate subnets; cache the image and checkpoint on every node; start workers before the head. Docker runs privileged.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.8-27B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.8-27B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-27b/dgx-spark/linux/vllm/128gb/nvfp4", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.31.0+88641915.nv26.10.73893335", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "nvfp4", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [ "The image is the public NGC vLLM 26.10.pre1 release (vLLM 0.31.0 with NVIDIA patches). It includes the SM12x NVFP4 dynamic linear backend (`nvfp4_dynamic`, vllm#54614), `unquantized_linear_backend` (vllm#59021), the cuDNN BF16 GEMM backend and its warmup (vllm#59025, vllm#59244) and the DFlash sliding-window draft block fix (vllm#60252), plus prebuilt kernel and autotune caches under `/root/.cache` (mount only `~/.cache/huggingface` as shown; mounting over all of `/root/.cache` hides them).", "Prefix caching (on by default) depends on vllm#60252. Without it, every prefix-cache hit with the DFlash2 drafter returns corrupted text (vllm#60174), so do not run this command on an image that lacks the fix.", "Validated with image input enabled on SPEED-Bench coding (concurrency 1, 8K output), the Artificial Analysis AgentPerf-Local default replay (168 multi-turn agentic turns, prefix caching on), a prefix-cache-hit correctness check and a single-image chat request.", "`--gpu-memory-utilization` is 0.60 on purpose: at 0.85 the node hung during startup autotune, and 0.70 took nodes down on some runs." ], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.8-27B-NVFP4", "target_label": "nvidia/Qwen3.8-27B-NVFP4", "draft": "https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2", "draft_label": "z-lab/Qwen3.8-27B-DFlash2" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container (NGC)", "image": "nvcr.io/nvidia/vllm:26.10.pre1-py3", "image_url": "https://catalog.ngc.nvidia.com/orgs/nvidia/containers/vllm", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment. The image is public on NGC, so no registry login is needed; if `docker pull` fails with `unauthorized`, clear a stale login with `docker logout nvcr.io`.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n nvcr.io/nvidia/vllm:26.10.pre1-py3" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Qwen3.8-27B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --kv-cache-dtype fp8 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --async-scheduling \\\n --enable-flashinfer-autotune \\\n --gpu-memory-utilization 0.60 \\\n --kernel-config '{\"linear_backend\":\"nvfp4_dynamic\",\"nvfp4_dynamic_max_tokens\":16,\"unquantized_linear_backend\":\"flashinfer_cudnn\",\"linear_backend_per_quant\":{\"fp8_w8a8\":\"cutlass\"}}' \\\n --speculative-config '{\"method\":\"dflash\",\"model\":\"z-lab/Qwen3.8-27B-DFlash2\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"standard\"}' \\\n --compilation-config '{\"pass_config\":{\"fuse_act_quant\":true},\"max_cudagraph_capture_size\":64}' \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice", "port": 8000, "api_model": "nvidia/Qwen3.8-27B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.8-27B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.8-27B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-27b/dgx-spark/linux/vllm/64gb/nvfp4", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.28.0", "backend_min_version": null, "min_vidmem_gb": 64, "precision": "nvfp4", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.8-27B-NVFP4", "target_label": "nvidia/Qwen3.8-27B-NVFP4", "draft": "https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2", "draft_label": "z-lab/Qwen3.8-27B-DFlash2" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:v0.28.0", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.28.0", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.28.0" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Qwen3.8-27B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --gpu-memory-utilization 0.8 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 16384 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"dflash\",\"model\": \"z-lab/Qwen3.8-27B-DFlash2\", \"num_speculative_tokens\":8}' \\\n --load-format safetensors \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice", "port": 8000, "api_model": "nvidia/Qwen3.8-27B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.8-27B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.8-27B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-27b/geforce-rtx/linux/llama-cpp/q4-k-m/mtp", "hardware": "geforce-rtx", "platform": { "os": "linux", "execution": "native" }, "backend": "llama-cpp", "backend_version": null, "backend_min_version": "b10505", "min_vidmem_gb": 22, "precision": "q4-k-m", "variant": "mtp", "status": "experimental", "prerequisites": [ "A Linux build of llama.cpp commit 7316c32d4f241820a5a4e12582f0f0b2b04938a7 providing llama-server." ], "limitations": [ "These settings are ported from the Windows 8K SPEED-Bench selection and have not been performance-validated on Linux." ], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/f1bfb127c64f7072bdd2cad55f258b9c8b2910fe/Qwen3.8-27B-Q4_K_M.gguf", "target_label": "unsloth/Qwen3.8-27B-GGUF", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": null, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in a Linux shell. llama-server downloads and caches the target GGUF from Hugging Face.", "command": "llama-server -hf \"unsloth/Qwen3.8-27B-GGUF\" \\\n -hff \"Qwen3.8-27B-Q4_K_M.gguf\" \\\n --no-mmproj \\\n --backend-sampling \\\n --batch-size 4096 \\\n --ubatch-size 2048 \\\n --ctx-size 65536 \\\n --parallel 1 \\\n --spec-type draft-mtp \\\n --spec-draft-n-max 2 \\\n --spec-draft-backend-sampling \\\n --temp 1.0 \\\n --top-p 0.95 \\\n --top-k 20 \\\n --min-p 0.0 \\\n --presence-penalty 0.0 \\\n --repeat-penalty 1.0", "port": null, "api_model": "unsloth/Qwen3.8-27B-GGUF" }, "launch": null, "sampling": "Target backend sampling: ON; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "The pinned [Q4_K_M target GGUF](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/f1bfb127c64f7072bdd2cad55f258b9c8b2910fe/Qwen3.8-27B-Q4_K_M.gguf) contains integrated MTP layers, so no separate draft checkpoint is required. The MTP depth and batch settings match the RTX 5090 Windows selection.", "clients": null }, { "id": "qwen/qwen3.8-27b/geforce-rtx/windows/llama-cpp/24gb/iq4-xs/vision-mtp", "hardware": "geforce-rtx", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b10746", "min_vidmem_gb": 24, "precision": "iq4-xs", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b10746, source commit d11b3cc7ed2b146a6045d55db30f46d511e25011, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/main/Qwen3.8-27B-UD-IQ4_XS.gguf", "target_label": "unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-IQ4_XS.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and projector files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/Qwen3.8-27B-GGUF\" `\n -hff \"Qwen3.8-27B-UD-IQ4_XS.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --backend-sampling `\n --batch-size 4096 `\n --ubatch-size 512 `\n --ctx-size 113664 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 2 `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.8-27B-GGUF" }, "launch": null, "sampling": "Target backend sampling: ON; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.8-27B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.8-27B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-27b/geforce-rtx/windows/llama-cpp/30gb/q4-k-m/vision-dflash2", "hardware": "geforce-rtx", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b10746", "min_vidmem_gb": 30, "precision": "q4-k-m", "variant": "vision-dflash2", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b10746, source commit d11b3cc7ed2b146a6045d55db30f46d511e25011, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/main/Qwen3.8-27B-UD-Q4_K_M.gguf", "target_label": "unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q4_K_M.gguf", "draft": "https://huggingface.co/incoai/Qwen3.8-27B-DFlash2-GGUF/blob/main/Qwen3.8-27B-DFlash2-Q4_K_M.gguf", "draft_label": "incoai/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch DFlash · n=7", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target, draft, and projector files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/Qwen3.8-27B-GGUF\" `\n -hff \"Qwen3.8-27B-UD-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --backend-sampling `\n --batch-size 4096 `\n --ubatch-size 512 `\n --ctx-size 163840 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 7 `\n -hfrd \"incoai/Qwen3.8-27B-DFlash2-GGUF\" `\n --spec-draft-model \"Qwen3.8-27B-DFlash2-Q4_K_M.gguf\" `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.8-27B-GGUF" }, "launch": null, "sampling": "Target backend sampling: ON; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.8-27B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.8-27B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-27b/rtx-pro/linux/llama-cpp/q4-k-m/mtp", "hardware": "rtx-pro", "platform": { "os": "linux", "execution": "native" }, "backend": "llama-cpp", "backend_version": null, "backend_min_version": "b10505", "min_vidmem_gb": 21.5, "precision": "q4-k-m", "variant": "mtp", "status": "experimental", "prerequisites": [ "A Linux build of llama.cpp commit 7316c32d4f241820a5a4e12582f0f0b2b04938a7 providing llama-server." ], "limitations": [ "These settings are ported from the Windows 8K SPEED-Bench selection and have not been performance-validated on Linux." ], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/f1bfb127c64f7072bdd2cad55f258b9c8b2910fe/Qwen3.8-27B-Q4_K_M.gguf", "target_label": "unsloth/Qwen3.8-27B-GGUF", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": null, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in a Linux shell. llama-server downloads and caches the target GGUF from Hugging Face.", "command": "llama-server -hf \"unsloth/Qwen3.8-27B-GGUF\" \\\n -hff \"Qwen3.8-27B-Q4_K_M.gguf\" \\\n --no-mmproj \\\n --batch-size 4096 \\\n --ubatch-size 1024 \\\n --ctx-size 65536 \\\n --parallel 1 \\\n --spec-type draft-mtp \\\n --spec-draft-n-max 3 \\\n --spec-draft-backend-sampling \\\n --temp 1.0 \\\n --top-p 0.95 \\\n --top-k 20 \\\n --min-p 0.0 \\\n --presence-penalty 0.0 \\\n --repeat-penalty 1.0", "port": null, "api_model": "unsloth/Qwen3.8-27B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "The pinned [Q4_K_M target GGUF](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/f1bfb127c64f7072bdd2cad55f258b9c8b2910fe/Qwen3.8-27B-Q4_K_M.gguf) contains integrated MTP layers, so no separate draft checkpoint is required. The MTP depth and batch settings match the RTX PRO 6000 Windows selection.", "clients": null }, { "id": "qwen/qwen3.8-27b/rtx-pro/linux/vllm/nvfp4", "hardware": "rtx-pro", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.31.0+88641915.nv26.10.73893335", "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [ "The image is the public NGC vLLM 26.10.pre1 release (vLLM 0.31.0 with NVIDIA patches). It includes the SM12x NVFP4 dynamic linear backend (`nvfp4_dynamic`, vllm#54614), `unquantized_linear_backend` (vllm#59021) and the DFlash sliding-window draft block fix (vllm#60252), plus prebuilt kernel and autotune caches under `/root/.cache` (mount only `~/.cache/huggingface` as shown; mounting over all of `/root/.cache` hides them).", "Prefix caching (on by default) depends on vllm#60252. Without it, every prefix-cache hit with the DFlash2 drafter returns corrupted text (vllm#60174), so do not run this command on an image that lacks the fix.", "Validated with image input enabled on SPEED-Bench coding (concurrency 1, 8K output), the Artificial Analysis AgentPerf-Local default replay (168 multi-turn agentic turns, prefix caching on), a prefix-cache-hit correctness check and a single-image chat request.", "The published performance figures were measured on the RTX PRO 6000 Blackwell Workstation Edition. The RTX PRO 6000 Blackwell Max-Q Workstation Edition (300 W) was used for functional validation only; its numbers are not the published ones." ], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.8-27B-NVFP4", "target_label": "nvidia/Qwen3.8-27B-NVFP4", "draft": "https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2", "draft_label": "z-lab/Qwen3.8-27B-DFlash2" }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× RTX Pro", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": null }, "runtime": { "kind": "container", "label": "Container (NGC)", "image": "nvcr.io/nvidia/vllm:26.10.pre1-py3", "image_url": "https://catalog.ngc.nvidia.com/orgs/nvidia/containers/vllm", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment. The image is public on NGC, so no registry login is needed; if `docker pull` fails with `unauthorized`, clear a stale login with `docker logout nvcr.io`.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n nvcr.io/nvidia/vllm:26.10.pre1-py3" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve nvidia/Qwen3.8-27B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --kv-cache-dtype fp8 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --async-scheduling \\\n --enable-flashinfer-autotune \\\n --gpu-memory-utilization 0.85 \\\n --kernel-config '{\"linear_backend\":\"nvfp4_dynamic\",\"nvfp4_dynamic_max_tokens\":16,\"linear_backend_per_quant\":{\"fp8_w8a8\":\"cutlass\"}}' \\\n --speculative-config '{\"method\":\"dflash\",\"model\":\"z-lab/Qwen3.8-27B-DFlash2\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"standard\"}' \\\n --compilation-config '{\"pass_config\":{\"fuse_act_quant\":true},\"max_cudagraph_capture_size\":64}' \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice", "port": 8000, "api_model": "nvidia/Qwen3.8-27B-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.8-27B-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.8-27B-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-27b/rtx-pro/windows/llama-cpp/24gb/iq4-xs/vision-mtp", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b10746", "min_vidmem_gb": 24, "precision": "iq4-xs", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/main/Qwen3.8-27B-UD-IQ4_XS.gguf", "target_label": "unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-IQ4_XS.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=2", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and projector files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/Qwen3.8-27B-GGUF\" `\n -hff \"Qwen3.8-27B-UD-IQ4_XS.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 1024 `\n --ctx-size 98304 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 2 `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.8-27B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.8-27B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.8-27B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-27b/rtx-pro/windows/llama-cpp/32gb/q4-k-m/vision-dflash2", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b10746", "min_vidmem_gb": 32, "precision": "q4-k-m", "variant": "vision-dflash2", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/main/Qwen3.8-27B-UD-Q4_K_M.gguf", "target_label": "unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q4_K_M.gguf", "draft": "https://huggingface.co/incoai/Qwen3.8-27B-DFlash2-GGUF/blob/main/Qwen3.8-27B-DFlash2-Q4_K_M.gguf", "draft_label": "incoai/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch DFlash · n=7", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target, draft, and projector files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/Qwen3.8-27B-GGUF\" `\n -hff \"Qwen3.8-27B-UD-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 1024 `\n --ctx-size 180224 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 7 `\n -hfrd \"incoai/Qwen3.8-27B-DFlash2-GGUF\" `\n --spec-draft-model \"Qwen3.8-27B-DFlash2-Q4_K_M.gguf\" `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.8-27B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.8-27B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.8-27B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-27b/rtx-pro/windows/llama-cpp/38gb/q4-k-m/vision-dflash2", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b10746", "min_vidmem_gb": 38, "precision": "q4-k-m", "variant": "vision-dflash2", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows x86-64 llama.cpp build b11425, source commit e117148a41d8e9bedb72e4c6c3f003ab0fe7f857, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/main/Qwen3.8-27B-UD-Q4_K_M.gguf", "target_label": "unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q4_K_M.gguf", "draft": "https://huggingface.co/incoai/Qwen3.8-27B-DFlash2-GGUF/blob/main/Qwen3.8-27B-DFlash2-Q4_K_M.gguf", "draft_label": "incoai/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch DFlash · n=7", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target, draft, and projector files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/Qwen3.8-27B-GGUF\" `\n -hff \"Qwen3.8-27B-UD-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 1024 `\n --ctx-size 262144 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 7 `\n -hfrd \"incoai/Qwen3.8-27B-DFlash2-GGUF\" `\n --spec-draft-model \"Qwen3.8-27B-DFlash2-Q4_K_M.gguf\" `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.8-27B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.8-27B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.8-27B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-27b/rtx-spark/windows/llama-cpp/20gb/iq4-xs/vision-mtp", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11609", "backend_min_version": "b11609", "min_vidmem_gb": 20, "precision": "iq4-xs", "variant": "vision-mtp", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build b11609, source commit fcac19079f3f2985fe74ed845f0880e7e9a8030a, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint and projector." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/main/Qwen3.8-27B-UD-IQ4_XS.gguf", "target_label": "unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-IQ4_XS.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11609. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11609\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and projector from Hugging Face. The MTP head ships inside the target GGUF, so no separate draft checkpoint is fetched.", "command": "llama-server.exe -hf \"unsloth/Qwen3.8-27B-GGUF\" `\n -hff \"Qwen3.8-27B-UD-IQ4_XS.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 512 `\n --ctx-size 98304 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-mtp `\n --spec-draft-n-max 3 `\n --spec-draft-backend-sampling `\n --cache-type-k q8_0 `\n --cache-type-v q8_0 `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.8-27B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "q8_0 K and V cache is what fits 96K context inside a 20 GB budget; every other Qwen3.8-27B recipe runs the default F16 cache. Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.8-27B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.8-27B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-27b/rtx-spark/windows/llama-cpp/24gb/iq4-xs/vision-dflash", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11572", "backend_min_version": "b11229", "min_vidmem_gb": 24, "precision": "iq4-xs", "variant": "vision-dflash", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build b11229, source commit c2a9e1606807970f4ee3167bacd951699c89caea, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/main/Qwen3.8-27B-UD-IQ4_XS.gguf", "target_label": "unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-IQ4_XS.gguf", "draft": "https://huggingface.co/incoai/Qwen3.8-27B-DFlash2-GGUF/blob/main/Qwen3.8-27B-DFlash2-Q4_K_M.gguf", "draft_label": "incoai/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11572. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11572\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch DFlash · n=7", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and draft files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/Qwen3.8-27B-GGUF\" `\n -hff \"Qwen3.8-27B-UD-IQ4_XS.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 512 `\n --ctx-size 105472 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 7 `\n -hfrd \"incoai/Qwen3.8-27B-DFlash2-GGUF\" `\n --spec-draft-model \"Qwen3.8-27B-DFlash2-Q4_K_M.gguf\" `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.8-27B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.8-27B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.8-27B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-27b/rtx-spark/windows/llama-cpp/28gb/q4-k-m/vision-dflash", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11572", "backend_min_version": "b11229", "min_vidmem_gb": 28, "precision": "q4-k-m", "variant": "vision-dflash", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build b11229, source commit c2a9e1606807970f4ee3167bacd951699c89caea, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/main/Qwen3.8-27B-UD-Q4_K_M.gguf", "target_label": "unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q4_K_M.gguf", "draft": "https://huggingface.co/incoai/Qwen3.8-27B-DFlash2-GGUF/blob/main/Qwen3.8-27B-DFlash2-Q4_K_M.gguf", "draft_label": "incoai/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11572. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11572\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch DFlash · n=7", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and draft files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/Qwen3.8-27B-GGUF\" `\n -hff \"Qwen3.8-27B-UD-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 512 `\n --ctx-size 138240 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 7 `\n -hfrd \"incoai/Qwen3.8-27B-DFlash2-GGUF\" `\n --spec-draft-model \"Qwen3.8-27B-DFlash2-Q4_K_M.gguf\" `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.8-27B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.8-27B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.8-27B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-27b/rtx-spark/windows/llama-cpp/36gb/q4-k-m/vision-dflash", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11572", "backend_min_version": "b11229", "min_vidmem_gb": 36, "precision": "q4-k-m", "variant": "vision-dflash", "status": "optimized", "prerequisites": [ "A CUDA-enabled Windows ARM64 llama.cpp build b11229, source commit c2a9e1606807970f4ee3167bacd951699c89caea, with llama-server.exe on PATH.", "Network access for llama-server to download and cache the named Hugging Face checkpoint files." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/main/Qwen3.8-27B-UD-Q4_K_M.gguf", "target_label": "unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q4_K_M.gguf", "draft": "https://huggingface.co/incoai/Qwen3.8-27B-DFlash2-GGUF/blob/main/Qwen3.8-27B-DFlash2-Q4_K_M.gguf", "draft_label": "incoai/Qwen3.8-27B-DFlash2-GGUF/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11572. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11572\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch DFlash · n=7", "target": "single-node", "instructions": "Run in PowerShell with llama-server.exe on PATH. llama-server downloads and caches the named target and draft files from Hugging Face. The omitted load-mode flag retains default mmap behavior.", "command": "llama-server.exe -hf \"unsloth/Qwen3.8-27B-GGUF\" `\n -hff \"Qwen3.8-27B-UD-Q4_K_M.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 4096 `\n --ubatch-size 512 `\n --ctx-size 262144 `\n --parallel 1 `\n -ngl 999 `\n --spec-type draft-dflash `\n --spec-draft-n-max 7 `\n -hfrd \"incoai/Qwen3.8-27B-DFlash2-GGUF\" `\n --spec-draft-model \"Qwen3.8-27B-DFlash2-Q4_K_M.gguf\" `\n --spec-draft-backend-sampling `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0", "port": 9931, "api_model": "unsloth/Qwen3.8-27B-GGUF" }, "launch": null, "sampling": "Target backend sampling: OFF; speculative draft backend sampling: ON; --temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Projector: [mmproj-F16.gguf](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/blob/4ca720788d1e01f1bff70c033e0d0028fd02e502/mmproj-F16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.8-27B-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.8-27B-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "qwen/qwen3.8-flash-next", "display_name": "Qwen3.8 Flash-Next", "builder": "qwen", "family": "Qwen3.8", "summary": "Qwen 3.8 (Flash-Next) text and vision model.", "upstream": null, "modalities": [ "text", "image" ], "recipes": [ { "id": "qwen/qwen3.8-flash-next/dgx-spark-2gpu/linux/vllm/128gb/fp4-local", "hardware": "dgx-spark-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21830+g8244d1c44.d20261015", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "fp4-local", "variant": null, "status": "optimized", "prerequisites": [ "Two DGX Spark nodes with 128 GB of memory per node and the ConnectX/RoCE network configured as described below.", "Docker with NVIDIA GPU support and the image and checkpoint cached on both nodes." ], "limitations": [ "The source requires the custom B12x container image." ], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.8-Flash-Next-FP4-Local", "target_label": "nvidia/Qwen3.8-Flash-Next-FP4-Local", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× DGX Spark; one node rank per Spark; tensor parallel 2", "gpu_count": 2, "node_count": 2, "tensor_parallel_size": 2 }, "runtime": { "kind": "container", "label": "Container (community, ARM64, custom B12x)", "image": "eugr/spark-vllm-b12x:nightly-20261015", "image_url": "https://hub.docker.com/layers/eugr/spark-vllm-b12x/nightly-20261015", "setup_label": "Start container", "setup_target": "all-nodes", "setup_instructions": "Run on every node after setting `HF_TOKEN`; replace the node-specific placeholders.", "setup_command": "# Example two-Spark values (adjust to your network):\n# HEAD_IP=\"10.100.32.1\" # head node IP on the ETH subnet\n# ETH_IF=\"enp1s0f0np0\" # ConnectX baseboard ethernet (socket/control)\n# IB_IF=\"rocep1s0f0,roceP2p1s0f0\" # RoCE HCA names (data plane) - use both \"twins\"!\nexport HEAD_IP=\"\"\nexport ETH_IF=\"\"\nexport IB_IF=\",\"\nexport MASTER_PORT=29501\nexport IMAGE=\"eugr/spark-vllm-b12x:nightly-20261015\"\nexport HF_CACHE_DIR=\"$HOME/.cache/huggingface\"\n\nmkdir -p \"$HF_CACHE_DIR\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\" \"$HOME/.cache/b12x\"\n\ndocker run --rm -it \\\n --privileged \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit nofile=1048576:1048576 \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HF_CACHE_DIR:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n -v $HOME/.cache/b12x:/root/.cache/b12x \\\n --entrypoint /bin/bash \\\n \"$IMAGE\"" }, "serve": { "label": "Launch head", "target": "head", "instructions": null, "command": "export VLLM_PLE_TABLE_MEMORY=\"shared\"\nexport VLLM_ENABLE_ROCE_ALLREDUCE=\"1\"\nexport VLLM_ROCE_ALLREDUCE_MAX_SIZE=\"2MB\"\nexport NCCL_BUFFSIZE=\"1048576\"\nexport NCCL_MAX_CTAS=\"8\"\nexport CUTE_DSL_ARCH=\"sm_121a\"\nexport SAFETENSORS_FAST_GPU=\"1\"\nexport VLLM_WORKER_MULTIPROC_METHOD=\"spawn\"\nexport VLLM_SSM_CONV_STATE_LAYOUT=\"DS\"\nexport VLLM_USE_AOT_COMPILE=\"1\"\nexport VLLM_USE_MEGA_AOT_ARTIFACT=\"1\"\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\nexport B12X_POLICY_MODE=\"auto\"\nexport VLLM_MXFP8_LM_HEAD=\"1\"\nexport VLLM_B12X_MOE_FP4_LAYER_MAX_INPUT_SCALE=\"w13\"\nexport VLLM_B12X_MOE_FP4_FORCE_A16=\"1\"\nexport B12X_W4A16_A4_PREFILL=\"1\"\n\nvllm serve nvidia/Qwen3.8-Flash-Next-FP4-Local \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --tensor-parallel-size 2 \\\n --pipeline-parallel-size 1 \\\n --mamba-cache-mode align \\\n --enable-prefix-caching \\\n --enable-chunked-prefill \\\n --dtype bfloat16 \\\n --kv-cache-dtype fp8 \\\n --quantization modelopt_mixed \\\n --block-size 16 \\\n --load-format b12x \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --max-num-batched-tokens 4096 \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4, \"draft_tensor_parallel_size\": 2 }' \\\n --gdn-decode-kernel b12x \\\n --linear-backend b12x \\\n --moe-backend b12x \\\n --no-enable-flashinfer-autotune \\\n --mm-encoder-tp-mode data \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice \\\n --compilation-config '{\"pass_config\":{\"fuse_act_quant\":true}}' \\\n --gpu-memory-utilization 0.7 \\\n --nnodes 2 \\\n --node-rank 0 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\"", "port": 8000, "api_model": "nvidia/Qwen3.8-Flash-Next-FP4-Local", "additional_commands": [ { "id": "worker", "label": "Launch worker", "target": "workers", "instructions": null, "command": "export VLLM_PLE_TABLE_MEMORY=\"shared\"\nexport VLLM_ENABLE_ROCE_ALLREDUCE=\"1\"\nexport VLLM_ROCE_ALLREDUCE_MAX_SIZE=\"2MB\"\nexport NCCL_BUFFSIZE=\"1048576\"\nexport NCCL_MAX_CTAS=\"8\"\nexport CUTE_DSL_ARCH=\"sm_121a\"\nexport SAFETENSORS_FAST_GPU=\"1\"\nexport VLLM_WORKER_MULTIPROC_METHOD=\"spawn\"\nexport VLLM_SSM_CONV_STATE_LAYOUT=\"DS\"\nexport VLLM_USE_AOT_COMPILE=\"1\"\nexport VLLM_USE_MEGA_AOT_ARTIFACT=\"1\"\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\nexport B12X_POLICY_MODE=\"auto\"\nexport VLLM_MXFP8_LM_HEAD=\"1\"\nexport VLLM_B12X_MOE_FP4_LAYER_MAX_INPUT_SCALE=\"w13\"\nexport VLLM_B12X_MOE_FP4_FORCE_A16=\"1\"\nexport B12X_W4A16_A4_PREFILL=\"1\"\n\nvllm serve nvidia/Qwen3.8-Flash-Next-FP4-Local \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --tensor-parallel-size 2 \\\n --pipeline-parallel-size 1 \\\n --mamba-cache-mode align \\\n --enable-prefix-caching \\\n --enable-chunked-prefill \\\n --dtype bfloat16 \\\n --kv-cache-dtype fp8 \\\n --quantization modelopt_mixed \\\n --block-size 16 \\\n --load-format b12x \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --max-num-batched-tokens 4096 \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4, \"draft_tensor_parallel_size\": 2 }' \\\n --gdn-decode-kernel b12x \\\n --linear-backend b12x \\\n --moe-backend b12x \\\n --no-enable-flashinfer-autotune \\\n --mm-encoder-tp-mode data \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice \\\n --compilation-config '{\"pass_config\":{\"fuse_act_quant\":true}}' \\\n --gpu-memory-utilization 0.7 \\\n --nnodes 2 \\\n --node-rank 1 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\" \\\n --headless" } ] }, "launch": null, "sampling": null, "notes": "Use the same CX7 port on every node; assign both port subinterfaces on separate subnets; cache the image and checkpoint on every node; start workers before the head. Docker runs privileged.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.8-Flash-Next-FP4-Local\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.8-Flash-Next-FP4-Local\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-flash-next/dgx-spark-2gpu/linux/vllm/64gb/fp4-local", "hardware": "dgx-spark-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21830+g8244d1c44.d20261015", "backend_min_version": null, "min_vidmem_gb": 64, "precision": "fp4-local", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [ "The source requires the custom B12x container image." ], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.8-Flash-Next-FP4-Local", "target_label": "nvidia/Qwen3.8-Flash-Next-FP4-Local", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× DGX Spark; one node rank per Spark; tensor parallel 2", "gpu_count": 2, "node_count": 2, "tensor_parallel_size": 2 }, "runtime": { "kind": "container", "label": "Container (community, ARM64, custom B12x)", "image": "eugr/spark-vllm-b12x:nightly-20261015", "image_url": "https://hub.docker.com/layers/eugr/spark-vllm-b12x/nightly-20261015", "setup_label": "Start container", "setup_target": "all-nodes", "setup_instructions": "Run on every node after setting `HF_TOKEN`; replace the node-specific placeholders.", "setup_command": "# Example two-Spark values (adjust to your network):\n# HEAD_IP=\"10.100.32.1\" # head node IP on the ETH subnet\n# ETH_IF=\"enp1s0f0np0\" # ConnectX baseboard ethernet (socket/control)\n# IB_IF=\"rocep1s0f0,roceP2p1s0f0\" # RoCE HCA names (data plane) - use both \"twins\"!\nexport HEAD_IP=\"\"\nexport ETH_IF=\"\"\nexport IB_IF=\",\"\nexport MASTER_PORT=29501\nexport IMAGE=\"eugr/spark-vllm-b12x:nightly-20261015\"\nexport HF_CACHE_DIR=\"$HOME/.cache/huggingface\"\n\nmkdir -p \"$HF_CACHE_DIR\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\" \"$HOME/.cache/b12x\"\n\ndocker run --rm -it \\\n --privileged \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit nofile=1048576:1048576 \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HF_CACHE_DIR:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n -v $HOME/.cache/b12x:/root/.cache/b12x \\\n --entrypoint /bin/bash \\\n \"$IMAGE\"" }, "serve": { "label": "Launch head", "target": "head", "instructions": null, "command": "export VLLM_PLE_TABLE_MEMORY=\"disk\"\nexport CUTE_DSL_ARCH=\"sm_121a\"\nexport SAFETENSORS_FAST_GPU=\"1\"\nexport VLLM_WORKER_MULTIPROC_METHOD=\"spawn\"\nexport VLLM_SSM_CONV_STATE_LAYOUT=\"DS\"\nexport VLLM_USE_AOT_COMPILE=\"1\"\nexport VLLM_USE_MEGA_AOT_ARTIFACT=\"1\"\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\nexport B12X_POLICY_MODE=\"auto\"\nexport VLLM_ENABLE_ROCE_ALLREDUCE=\"1\"\nexport VLLM_ROCE_ALLREDUCE_MAX_SIZE=\"2MB\"\nexport NCCL_BUFFSIZE=\"1048576\"\nexport NCCL_MAX_CTAS=\"8\"\nexport B12X_WEIGHTS_COMPILE_WORKERS=\"4\"\nexport B12X_STATE_COMPILE_WORKERS=\"4\"\nexport B12X_BIND_COMPILE_WORKERS=\"4\"\nexport VLLM_B12X_MOE_FP4_FORCE_A16=\"1\"\nexport B12X_W4A16_A4_PREFILL=\"1\"\n\nvllm serve nvidia/Qwen3.8-Flash-Next-FP4-Local \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --tensor-parallel-size 2 \\\n --mamba-cache-mode align \\\n --enable-prefix-caching \\\n --enable-chunked-prefill \\\n --dtype bfloat16 \\\n --kv-cache-dtype fp8 \\\n --quantization modelopt_mixed \\\n --block-size 16 \\\n --load-format b12x \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 2048 \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4}' \\\n --gdn-decode-kernel b12x \\\n --linear-backend b12x \\\n --moe-backend b12x \\\n --no-enable-flashinfer-autotune \\\n --mm-encoder-tp-mode data \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice \\\n --compilation-config '{\"pass_config\":{\"fuse_act_quant\":true},\"max_cudagraph_capture_size\":40,\"cudagraph_capture_sizes\":[1,2,4,5,8,10,16,20,24,32,40]}' \\\n --mm-processor-cache-gb 0 \\\n --limit-mm-per-prompt '{\"image\":1}' \\\n --gpu-memory-utilization 0.8 \\\n --kv-cache-memory-bytes 3G \\\n --nnodes 2 \\\n --node-rank 0 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\"", "port": 8000, "api_model": "nvidia/Qwen3.8-Flash-Next-FP4-Local", "additional_commands": [ { "id": "worker", "label": "Launch worker", "target": "workers", "instructions": null, "command": "export VLLM_PLE_TABLE_MEMORY=\"disk\"\nexport CUTE_DSL_ARCH=\"sm_121a\"\nexport SAFETENSORS_FAST_GPU=\"1\"\nexport VLLM_WORKER_MULTIPROC_METHOD=\"spawn\"\nexport VLLM_SSM_CONV_STATE_LAYOUT=\"DS\"\nexport VLLM_USE_AOT_COMPILE=\"1\"\nexport VLLM_USE_MEGA_AOT_ARTIFACT=\"1\"\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\nexport B12X_POLICY_MODE=\"auto\"\nexport VLLM_ENABLE_ROCE_ALLREDUCE=\"1\"\nexport VLLM_ROCE_ALLREDUCE_MAX_SIZE=\"2MB\"\nexport NCCL_BUFFSIZE=\"1048576\"\nexport NCCL_MAX_CTAS=\"8\"\nexport B12X_WEIGHTS_COMPILE_WORKERS=\"4\"\nexport B12X_STATE_COMPILE_WORKERS=\"4\"\nexport B12X_BIND_COMPILE_WORKERS=\"4\"\nexport VLLM_B12X_MOE_FP4_FORCE_A16=\"1\"\nexport B12X_W4A16_A4_PREFILL=\"1\"\n\nvllm serve nvidia/Qwen3.8-Flash-Next-FP4-Local \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --tensor-parallel-size 2 \\\n --mamba-cache-mode align \\\n --enable-prefix-caching \\\n --enable-chunked-prefill \\\n --dtype bfloat16 \\\n --kv-cache-dtype fp8 \\\n --quantization modelopt_mixed \\\n --block-size 16 \\\n --load-format b12x \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 2048 \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4}' \\\n --gdn-decode-kernel b12x \\\n --linear-backend b12x \\\n --moe-backend b12x \\\n --no-enable-flashinfer-autotune \\\n --mm-encoder-tp-mode data \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice \\\n --compilation-config '{\"pass_config\":{\"fuse_act_quant\":true},\"max_cudagraph_capture_size\":40,\"cudagraph_capture_sizes\":[1,2,4,5,8,10,16,20,24,32,40]}' \\\n --mm-processor-cache-gb 0 \\\n --limit-mm-per-prompt '{\"image\":1}' \\\n --gpu-memory-utilization 0.8 \\\n --kv-cache-memory-bytes 3G \\\n --nnodes 2 \\\n --node-rank 1 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\" \\\n --headless" } ] }, "launch": null, "sampling": null, "notes": "Use the same CX7 port on every node; assign both port subinterfaces on separate subnets; cache the image and checkpoint on every node; start workers before the head. Docker runs privileged.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.8-Flash-Next-FP4-Local\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.8-Flash-Next-FP4-Local\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-flash-next/dgx-spark/linux/vllm/fp4-local", "hardware": "dgx-spark", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21830+g8244d1c44.d20261015", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "fp4-local", "variant": null, "status": "optimized", "prerequisites": [ "A single DGX Spark with 128 GB of memory.", "Docker with NVIDIA GPU support and access to the target checkpoint." ], "limitations": [ "The source requires the custom B12x container image." ], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.8-Flash-Next-FP4-Local", "target_label": "nvidia/Qwen3.8-Flash-Next-FP4-Local", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Spark", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container (community, ARM64, custom B12x)", "image": "eugr/spark-vllm-b12x:nightly-20261015", "image_url": "https://hub.docker.com/layers/eugr/spark-vllm-b12x/nightly-20261015", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "mkdir -p $HOME/.cache/huggingface $HOME/.cache/vllm $HOME/.cache/flashinfer $HOME/.triton $HOME/.cache/b12x\n\ndocker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n --security-opt seccomp=unconfined \\\n --security-opt apparmor=unconfined \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v $HOME/.cache/vllm:/root/.cache/vllm \\\n -v $HOME/.cache/flashinfer:/root/.cache/flashinfer \\\n -v $HOME/.triton:/root/.triton \\\n -v $HOME/.cache/b12x:/root/.cache/b12x \\\n --entrypoint /bin/bash \\\n eugr/spark-vllm-b12x:nightly-20261015" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "export VLLM_PLE_TABLE_MEMORY=\"disk\"\nexport B12X_DISK_BACKEND=\"io_uring\"\nexport CUTE_DSL_ARCH=\"sm_121a\"\nexport SAFETENSORS_FAST_GPU=\"1\"\nexport VLLM_WORKER_MULTIPROC_METHOD=\"spawn\"\nexport VLLM_SSM_CONV_STATE_LAYOUT=\"DS\"\nexport VLLM_USE_AOT_COMPILE=\"1\"\nexport VLLM_USE_MEGA_AOT_ARTIFACT=\"1\"\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\nexport B12X_POLICY_MODE=\"auto\"\nexport VLLM_MXFP8_LM_HEAD=\"1\"\nexport VLLM_B12X_MOE_FP4_LAYER_MAX_INPUT_SCALE=\"w13\"\nexport VLLM_B12X_MOE_FP4_FORCE_A16=1\nexport B12X_W4A16_A4_PREFILL=1\n\nvllm serve nvidia/Qwen3.8-Flash-Next-FP4-Local \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --tensor-parallel-size 1 \\\n --pipeline-parallel-size 1 \\\n --mamba-cache-mode align \\\n --enable-prefix-caching \\\n --enable-chunked-prefill \\\n --dtype bfloat16 \\\n --kv-cache-dtype fp8 \\\n --quantization modelopt_mixed \\\n --block-size 16 \\\n --load-format b12x \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 4096 \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4}' \\\n --gdn-decode-kernel b12x \\\n --linear-backend b12x \\\n --moe-backend b12x \\\n --no-enable-flashinfer-autotune \\\n --mm-encoder-tp-mode data \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_xml \\\n --enable-auto-tool-choice \\\n --compilation-config '{\"pass_config\":{\"fuse_act_quant\":true}}' \\\n --gpu-memory-utilization 0.8", "port": 8000, "api_model": "nvidia/Qwen3.8-Flash-Next-FP4-Local" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.8-Flash-Next-FP4-Local\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.8-Flash-Next-FP4-Local\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-flash-next/dgx-station/linux/vllm/nvfp4", "hardware": "dgx-station", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": null, "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4", "variant": null, "status": "experimental", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/Inferact/Qwen3.8-Flash-Next-NVFP4", "target_label": "Inferact/Qwen3.8-Flash-Next-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Station", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container", "image": "vllm/vllm-openai:qwen38-flash-next", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=qwen38-flash-next", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:qwen38-flash-next" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "vllm serve Inferact/Qwen3.8-Flash-Next-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --no-enable-flashinfer-autotune \\\n --max-num-seqs 32 \\\n --gpu-memory-utilization 0.95 \\\n --max-num-batched-tokens 8192 \\\n --enable-prefix-caching \\\n --distributed-executor-backend mp \\\n --compilation-config '{\"mode\": \"none\", \"cudagraph_mode\": \"full_decode_only\"}' \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_xml \\\n --reasoning-parser qwen3", "port": 8000, "api_model": "Inferact/Qwen3.8-Flash-Next-NVFP4" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"Inferact/Qwen3.8-Flash-Next-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"Inferact/Qwen3.8-Flash-Next-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-flash-next/rtx-pro-2gpu/linux/vllm/fp4-local", "hardware": "rtx-pro-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21830+g8244d1c44.cu134", "backend_min_version": null, "min_vidmem_gb": 96, "precision": "fp4-local", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.8-Flash-Next-FP4-Local", "target_label": "nvidia/Qwen3.8-Flash-Next-FP4-Local", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× RTX Pro 6000", "gpu_count": 2, "node_count": 1, "tensor_parallel_size": 2 }, "runtime": { "kind": "container", "label": "Container (community)", "image": "ghcr.io/local-inference-lab/vllm:release-10152026", "image_url": "https://github.com/local-inference-lab/blackwell-llm-docker/pkgs/container/vllm/1392689728?tag=release-10152026", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n ghcr.io/local-inference-lab/vllm:release-10152026" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "export B12X_DENSE_SPLITK_TURBO=1\nexport B12X_DYNAMIC_DETERMINISTIC_OUTPUT=0\nexport B12X_DYNAMIC_DIRECT_EXPERT_SCALES=1\nexport B12X_DYNAMIC_SKIP_SPLIT_BARRIER_RESET=1\nexport B12X_DYNAMIC_SPLIT_COMPUTE_MAC=224\nexport B12X_DYNAMIC_SPLIT_FAST_PREPARE=1\nexport B12X_DYNAMIC_SPLIT_LOW_SMEM=1\nexport B12X_DYNAMIC_SPLIT_ROUTE_COMPUTE=1\nexport B12X_DYNAMIC_WORK_SOURCE=persistent_grid\nexport B12X_W4A16_A4_PREFILL=1\nexport B12X_DISK_BACKEND=io_uring\nexport B12X_MHC_PDL=1\nexport B12X_PCIE_ONESHOT_BLOCK_LIMIT=4\nexport B12X_PCIE_ONESHOT_PDL=1\nexport B12X_PCIE_ONESHOT_THREADS=512\nexport CUDA_BINARY_LOADER_THREAD_COUNT=8\nexport CUDA_DEVICE_ORDER=PCI_BUS_ID\nexport CUDA_MODULE_LOADING=LAZY\nexport CUTE_DSL_ARCH=sm_120a\nexport INSTANTTENSOR_BACKEND=BUFFERED\nexport NCCL_BUFFSIZE=2097152\nexport NCCL_CUMEM_ENABLE=0\nexport NCCL_IB_DISABLE=1\nexport NCCL_MAX_NCHANNELS=16\nexport NCCL_MIN_NCHANNELS=16\nexport NCCL_NET_PLUGIN=spcx\nexport NCCL_P2P_LEVEL=SYS\nexport NCCL_PROTO=LL,LL128,Simple\nexport OMP_NUM_THREADS=2\nexport PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport SAFETENSORS_FAST_GPU=1\nexport TORCHINDUCTOR_LOOP_ORDERING_AFTER_FUSION=0\nexport TORCH_CUDA_ARCH_LIST=12.0a\nexport VLLM_B12X_DENSE_ACTIVATION_MODE=auto\nexport VLLM_B12X_MOE_FP4_FORCE_A16=1\nexport VLLM_CAUSAL_CONV1D_UPDATE_HOIST=1\nexport VLLM_COMPUTE_NANS_IN_LOGITS=0\nexport VLLM_DISABLED_KERNELS=MarlinFP8ScaledMMLinearKernel\nexport VLLM_ENABLE_PCIE_ALLREDUCE=1\nexport VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH=1\nexport VLLM_LM_HEAD_A16=1\nexport VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0\nexport VLLM_MTP_NVFP4_LM_HEAD=1\nexport VLLM_MXFP8_LM_HEAD=0\nexport VLLM_PCIE_ALLREDUCE_BACKEND=b12x\nexport VLLM_PLE_CPU_OFFLOAD=1\nexport VLLM_QWEN3_8_FLASH_NEXT_HC_TP=0\nexport VLLM_QWEN3_8_FLASH_NEXT_MTP_COMPACT=1\nexport VLLM_QWEN3_8_FLASH_NEXT_OVERLAP=1\nexport VLLM_SSM_CONV_STATE_LAYOUT=DS\nexport VLLM_USE_FASTOKENS=1\nexport VLLM_USE_V2_MODEL_RUNNER=1\nexport VLLM_WORKER_MULTIPROC_METHOD=spawn\nexport TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1\nexport TORCH_NCCL_USE_COMM_NONBLOCKING=0\n\nvllm serve nvidia/Qwen3.8-Flash-Next-FP4-Local \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --async-scheduling \\\n --block-size 64 \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}' \\\n --decode-context-parallel-size 1 \\\n --dtype bfloat16 \\\n --enable-auto-tool-choice \\\n --enable-chunked-prefill \\\n --no-enable-flashinfer-autotune \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --gdn-decode-kernel b12x \\\n --gpu-memory-utilization 0.92 \\\n --kv-cache-dtype fp8 \\\n --no-language-model-only \\\n --linear-backend b12x \\\n --load-format instanttensor \\\n --mamba-cache-mode align \\\n --mamba-ssm-cache-dtype auto \\\n --max-cudagraph-capture-size 64 \\\n --max-model-len 262144 \\\n --max-num-batched-tokens 8192 \\\n --max-num-seqs 16 \\\n --mm-encoder-tp-mode data \\\n --mm-processor-cache-gb 0.0 \\\n --moe-backend b12x \\\n --pipeline-parallel-size 1 \\\n --quantization modelopt_mixed \\\n --reasoning-parser qwen3 \\\n --speculative-config '{\"method\":\"mtp\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"standard\",\"moe_backend\":\"b12x\",\"num_speculative_tokens\":3}' \\\n --tensor-parallel-size 2 \\\n --tool-call-parser qwen3_xml", "port": 8000, "api_model": "nvidia/Qwen3.8-Flash-Next-FP4-Local" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.8-Flash-Next-FP4-Local\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.8-Flash-Next-FP4-Local\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-flash-next/rtx-pro-4gpu/linux/vllm/fp4-local", "hardware": "rtx-pro-4gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21830+g8244d1c44.cu134", "backend_min_version": null, "min_vidmem_gb": 96, "precision": "fp4-local", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.8-Flash-Next-FP4-Local", "target_label": "nvidia/Qwen3.8-Flash-Next-FP4-Local", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "4× RTX Pro 6000", "gpu_count": 4, "node_count": 1, "tensor_parallel_size": 4 }, "runtime": { "kind": "container", "label": "Container (community)", "image": "ghcr.io/local-inference-lab/vllm:release-10152026", "image_url": "https://github.com/local-inference-lab/blackwell-llm-docker/pkgs/container/vllm/1392689728?tag=release-10152026", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n ghcr.io/local-inference-lab/vllm:release-10152026" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "export B12X_DENSE_SPLITK_TURBO=1\nexport B12X_DYNAMIC_DETERMINISTIC_OUTPUT=0\nexport B12X_DYNAMIC_DIRECT_EXPERT_SCALES=1\nexport B12X_DYNAMIC_SKIP_SPLIT_BARRIER_RESET=1\nexport B12X_DYNAMIC_SPLIT_COMPUTE_MAC=224\nexport B12X_DYNAMIC_SPLIT_FAST_PREPARE=1\nexport B12X_DYNAMIC_SPLIT_LOW_SMEM=1\nexport B12X_DYNAMIC_SPLIT_ROUTE_COMPUTE=1\nexport B12X_DYNAMIC_WORK_SOURCE=persistent_grid\nexport B12X_W4A16_A4_PREFILL=1\nexport B12X_DISK_BACKEND=io_uring\nexport B12X_MHC_PDL=1\nexport B12X_PCIE_ONESHOT_BLOCK_LIMIT=4\nexport B12X_PCIE_ONESHOT_PDL=1\nexport B12X_PCIE_ONESHOT_THREADS=512\nexport CUDA_BINARY_LOADER_THREAD_COUNT=8\nexport CUDA_DEVICE_ORDER=PCI_BUS_ID\nexport CUDA_MODULE_LOADING=LAZY\nexport CUTE_DSL_ARCH=sm_120a\nexport INSTANTTENSOR_BACKEND=BUFFERED\nexport NCCL_BUFFSIZE=2097152\nexport NCCL_CUMEM_ENABLE=0\nexport NCCL_IB_DISABLE=1\nexport NCCL_MAX_NCHANNELS=16\nexport NCCL_MIN_NCHANNELS=16\nexport NCCL_NET_PLUGIN=spcx\nexport NCCL_P2P_LEVEL=SYS\nexport NCCL_PROTO=LL,LL128,Simple\nexport OMP_NUM_THREADS=2\nexport PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport SAFETENSORS_FAST_GPU=1\nexport TORCHINDUCTOR_LOOP_ORDERING_AFTER_FUSION=0\nexport TORCH_CUDA_ARCH_LIST=12.0a\nexport VLLM_B12X_DENSE_ACTIVATION_MODE=auto\nexport VLLM_B12X_MOE_FP4_FORCE_A16=1\nexport VLLM_CAUSAL_CONV1D_UPDATE_HOIST=1\nexport VLLM_COMPUTE_NANS_IN_LOGITS=0\nexport VLLM_DISABLED_KERNELS=MarlinFP8ScaledMMLinearKernel\nexport VLLM_ENABLE_PCIE_ALLREDUCE=1\nexport VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH=1\nexport VLLM_LM_HEAD_A16=1\nexport VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0\nexport VLLM_MTP_NVFP4_LM_HEAD=1\nexport VLLM_MXFP8_LM_HEAD=0\nexport VLLM_PCIE_ALLREDUCE_BACKEND=b12x\nexport VLLM_PLE_CPU_OFFLOAD=1\nexport VLLM_QWEN3_8_FLASH_NEXT_HC_TP=0\nexport VLLM_QWEN3_8_FLASH_NEXT_MTP_COMPACT=1\nexport VLLM_QWEN3_8_FLASH_NEXT_OVERLAP=1\nexport VLLM_SSM_CONV_STATE_LAYOUT=DS\nexport VLLM_USE_FASTOKENS=1\nexport VLLM_USE_V2_MODEL_RUNNER=1\nexport VLLM_WORKER_MULTIPROC_METHOD=spawn\nexport TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1\nexport TORCH_NCCL_USE_COMM_NONBLOCKING=0\n\nvllm serve nvidia/Qwen3.8-Flash-Next-FP4-Local \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --async-scheduling \\\n --block-size 64 \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}' \\\n --decode-context-parallel-size 1 \\\n --dtype bfloat16 \\\n --enable-auto-tool-choice \\\n --enable-chunked-prefill \\\n --no-enable-flashinfer-autotune \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --gdn-decode-kernel b12x \\\n --gpu-memory-utilization 0.92 \\\n --kv-cache-dtype fp8 \\\n --no-language-model-only \\\n --linear-backend b12x \\\n --load-format instanttensor \\\n --mamba-cache-mode align \\\n --mamba-ssm-cache-dtype auto \\\n --max-cudagraph-capture-size 64 \\\n --max-model-len 262144 \\\n --max-num-batched-tokens 8192 \\\n --max-num-seqs 16 \\\n --mm-encoder-tp-mode data \\\n --mm-processor-cache-gb 0.0 \\\n --moe-backend b12x \\\n --pipeline-parallel-size 1 \\\n --quantization modelopt_mixed \\\n --reasoning-parser qwen3 \\\n --speculative-config '{\"method\":\"mtp\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"standard\",\"moe_backend\":\"b12x\",\"num_speculative_tokens\":3}' \\\n --tensor-parallel-size 4 \\\n --tool-call-parser qwen3_xml", "port": 8000, "api_model": "nvidia/Qwen3.8-Flash-Next-FP4-Local" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.8-Flash-Next-FP4-Local\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.8-Flash-Next-FP4-Local\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-flash-next/rtx-pro/linux/vllm/fp4-local", "hardware": "rtx-pro", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21830+g8244d1c44", "backend_min_version": null, "min_vidmem_gb": 96, "precision": "fp4-local", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/Qwen3.8-Flash-Next-FP4-Local", "target_label": "nvidia/Qwen3.8-Flash-Next-FP4-Local", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× RTX Pro", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container (community)", "image": "ghcr.io/local-inference-lab/vllm:release-10152026", "image_url": "https://github.com/local-inference-lab/blackwell-llm-docker/pkgs/container/vllm/1392689728?tag=release-10152026", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n ghcr.io/local-inference-lab/vllm:release-10152026" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "export B12X_DENSE_SPLITK_TURBO=1\nexport B12X_DYNAMIC_DETERMINISTIC_OUTPUT=0\nexport B12X_DYNAMIC_DIRECT_EXPERT_SCALES=1\nexport B12X_DYNAMIC_SKIP_SPLIT_BARRIER_RESET=1\nexport B12X_DYNAMIC_SPLIT_COMPUTE_MAC=224\nexport B12X_DYNAMIC_SPLIT_FAST_PREPARE=1\nexport B12X_DYNAMIC_SPLIT_LOW_SMEM=1\nexport B12X_DYNAMIC_SPLIT_ROUTE_COMPUTE=1\nexport B12X_DYNAMIC_WORK_SOURCE=persistent_grid\nexport B12X_W4A16_A4_PREFILL=1\nexport B12X_DISK_BACKEND=io_uring\nexport B12X_MHC_PDL=1\nexport B12X_PCIE_ONESHOT_BLOCK_LIMIT=4\nexport B12X_PCIE_ONESHOT_PDL=1\nexport B12X_PCIE_ONESHOT_THREADS=512\nexport CUDA_BINARY_LOADER_THREAD_COUNT=8\nexport CUDA_DEVICE_ORDER=PCI_BUS_ID\nexport CUDA_MODULE_LOADING=LAZY\nexport CUTE_DSL_ARCH=sm_120a\nexport INSTANTTENSOR_BACKEND=BUFFERED\nexport NCCL_BUFFSIZE=2097152\nexport NCCL_CUMEM_ENABLE=0\nexport NCCL_IB_DISABLE=1\nexport NCCL_MAX_NCHANNELS=16\nexport NCCL_MIN_NCHANNELS=16\nexport NCCL_NET_PLUGIN=spcx\nexport NCCL_P2P_LEVEL=SYS\nexport NCCL_PROTO=LL,LL128,Simple\nexport OMP_NUM_THREADS=2\nexport PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True\nexport SAFETENSORS_FAST_GPU=1\nexport TORCHINDUCTOR_LOOP_ORDERING_AFTER_FUSION=0\nexport TORCH_CUDA_ARCH_LIST=12.0a\nexport VLLM_B12X_DENSE_ACTIVATION_MODE=auto\nexport VLLM_B12X_MOE_FP4_FORCE_A16=1\nexport VLLM_CAUSAL_CONV1D_UPDATE_HOIST=1\nexport VLLM_COMPUTE_NANS_IN_LOGITS=0\nexport VLLM_DISABLED_KERNELS=MarlinFP8ScaledMMLinearKernel\nexport VLLM_ENABLE_PCIE_ALLREDUCE=1\nexport VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH=1\nexport VLLM_LM_HEAD_A16=1\nexport VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0\nexport VLLM_MTP_NVFP4_LM_HEAD=1\nexport VLLM_MXFP8_LM_HEAD=0\nexport VLLM_PCIE_ALLREDUCE_BACKEND=b12x\nexport VLLM_PLE_CPU_OFFLOAD=1\nexport VLLM_QWEN3_8_FLASH_NEXT_HC_TP=0\nexport VLLM_QWEN3_8_FLASH_NEXT_MTP_COMPACT=1\nexport VLLM_QWEN3_8_FLASH_NEXT_OVERLAP=1\nexport VLLM_SSM_CONV_STATE_LAYOUT=DS\nexport VLLM_USE_FASTOKENS=1\nexport VLLM_USE_V2_MODEL_RUNNER=1\nexport VLLM_WORKER_MULTIPROC_METHOD=spawn\nexport TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1\nexport TORCH_NCCL_USE_COMM_NONBLOCKING=0\nvllm serve nvidia/Qwen3.8-Flash-Next-FP4-Local \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --async-scheduling \\\n --block-size 64 \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}' \\\n --decode-context-parallel-size 1 \\\n --dtype bfloat16 \\\n --enable-auto-tool-choice \\\n --enable-chunked-prefill \\\n --no-enable-flashinfer-autotune \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --gdn-decode-kernel b12x \\\n --gpu-memory-utilization 0.975 \\\n --kv-cache-dtype fp8 \\\n --no-language-model-only \\\n --linear-backend b12x \\\n --load-format instanttensor \\\n --mamba-cache-mode align \\\n --mamba-ssm-cache-dtype auto \\\n --max-cudagraph-capture-size 64 \\\n --max-model-len 262144 \\\n --max-num-batched-tokens 6019 \\\n --max-num-seqs 16 \\\n --mm-encoder-tp-mode data \\\n --mm-processor-cache-gb 0.0 \\\n --moe-backend b12x \\\n --pipeline-parallel-size 1 \\\n --quantization modelopt_mixed \\\n --reasoning-parser qwen3 \\\n --speculative-config '{\"method\":\"mtp\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"standard\",\"moe_backend\":\"b12x\",\"num_speculative_tokens\":3}' \\\n --tensor-parallel-size 1 \\\n --tool-call-parser qwen3_xml", "port": 8000, "api_model": "nvidia/Qwen3.8-Flash-Next-FP4-Local" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/Qwen3.8-Flash-Next-FP4-Local\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/Qwen3.8-Flash-Next-FP4-Local\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-flash-next/rtx-pro/windows/llama-cpp/78gb/iq4-xs/vision-mtp", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11574", "backend_min_version": "b11370", "min_vidmem_gb": 78, "precision": "iq4-xs", "variant": "vision-mtp", "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF/blob/main/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf", "target_label": "unsloth/Qwen3.8-Flash-Next-GGUF/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf", "draft": "https://huggingface.co/ggml-org/Qwen3.8-Flash-Next-GGUF/blob/main/mtp-Qwen3.8-Flash-Next-Q8_0.gguf", "draft_label": "ggml-org Q8_0 MTP head (main model remains UD-IQ4_XS)" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11574. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11574\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in PowerShell from the extracted b11370 directory. llama-server downloads the specified target shards, Q8_0 MTP head and BF16 projector using its normal cache.", "command": ".\\llama-server.exe -hf \"unsloth/Qwen3.8-Flash-Next-GGUF\" `\n -hff \"UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF/resolve/38bb39ee97821de2c9009abb7e93950eec396e66/mmproj-BF16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 2048 `\n --ubatch-size 512 `\n --ctx-size 262144 `\n --parallel 1 `\n -fa on `\n -lzm on `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0 `\n --spec-type draft-mtp `\n --spec-draft-n-max 3 `\n -hfrd \"ggml-org/Qwen3.8-Flash-Next-GGUF\" `\n --spec-draft-model \"mtp-Qwen3.8-Flash-Next-Q8_0.gguf\"", "port": 9931, "api_model": "unsloth/Qwen3.8-Flash-Next-GGUF" }, "launch": null, "sampling": "--temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Target shards: [1](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF/blob/main/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf), [2](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF/blob/main/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00002-of-00003.gguf), [3](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF/blob/main/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00003-of-00003.gguf).\nMTP head: [mtp-Qwen3.8-Flash-Next-Q8_0.gguf](https://huggingface.co/ggml-org/Qwen3.8-Flash-Next-GGUF/blob/main/mtp-Qwen3.8-Flash-Next-Q8_0.gguf).\nProjector: [mmproj-BF16.gguf](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF/blob/38bb39ee97821de2c9009abb7e93950eec396e66/mmproj-BF16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.8-Flash-Next-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.8-Flash-Next-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "qwen/qwen3.8-flash-next/rtx-spark/windows/llama-cpp/78gb/iq4-xs/vision-mtp", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11617", "backend_min_version": "b11370", "min_vidmem_gb": 78, "precision": "iq4-xs", "variant": "vision-mtp", "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF/blob/main/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf", "target_label": "unsloth/Qwen3.8-Flash-Next-GGUF/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf", "draft": "https://huggingface.co/ggml-org/Qwen3.8-Flash-Next-GGUF/blob/main/mtp-Qwen3.8-Flash-Next-Q8_0.gguf", "draft_label": "ggml-org Q8_0 MTP head (main model remains UD-IQ4_XS)" }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11617. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11617\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch MTP · n=3", "target": "single-node", "instructions": "Run in PowerShell from the extracted b11617 directory. llama-server downloads the specified target shards, Q8_0 MTP head and BF16 projector using its normal cache.", "command": ".\\llama-server.exe -hf \"unsloth/Qwen3.8-Flash-Next-GGUF\" `\n -hff \"UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf\" `\n --mmproj-url \"https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF/resolve/38bb39ee97821de2c9009abb7e93950eec396e66/mmproj-BF16.gguf\" `\n --host 127.0.0.1 `\n --port 9931 `\n --batch-size 2048 `\n --ubatch-size 512 `\n --ctx-size 262144 `\n --parallel 1 `\n -fa on `\n -lzm on `\n --temp 1.0 `\n --top-p 0.95 `\n --top-k 20 `\n --min-p 0.0 `\n --presence-penalty 0.0 `\n --repeat-penalty 1.0 `\n --spec-type draft-mtp `\n --spec-draft-n-max 3 `\n -hfrd \"ggml-org/Qwen3.8-Flash-Next-GGUF\" `\n --spec-draft-model \"mtp-Qwen3.8-Flash-Next-Q8_0.gguf\"", "port": 9931, "api_model": "unsloth/Qwen3.8-Flash-Next-GGUF" }, "launch": null, "sampling": "--temp=1.0; --top-p=0.95; --top-k=20; --min-p=0.0; --presence-penalty=0.0; --repeat-penalty=1.0", "notes": "Target shards: [1](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF/blob/main/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf), [2](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF/blob/main/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00002-of-00003.gguf), [3](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF/blob/main/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00003-of-00003.gguf).\nMTP head: [mtp-Qwen3.8-Flash-Next-Q8_0.gguf](https://huggingface.co/ggml-org/Qwen3.8-Flash-Next-GGUF/blob/main/mtp-Qwen3.8-Flash-Next-Q8_0.gguf).\nProjector: [mmproj-BF16.gguf](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF/blob/38bb39ee97821de2c9009abb7e93950eec396e66/mmproj-BF16.gguf).", "clients": { "shell": "'{\"model\":\"unsloth/Qwen3.8-Flash-Next-GGUF\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:9931/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:9931/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"unsloth/Qwen3.8-Flash-Next-GGUF\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "recursivesi/flint", "display_name": "Recursive Flint 40B-A2.5B", "builder": "recursivesi", "family": "Flint", "summary": "Recursive Flint mixture-of-experts model with 39.8B total parameters and approximately 2.5B active parameters per token, including the output head.", "upstream": "https://huggingface.co/RecursiveSI/flint", "modalities": [ "text" ], "recipes": [ { "id": "recursivesi/flint/geforce-rtx/windows/llama-cpp/q4-1", "hardware": "geforce-rtx", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11803", "backend_min_version": "b11803", "min_vidmem_gb": 28, "precision": "q4-1", "variant": null, "status": "experimental", "prerequisites": [ "Windows x64 CUDA 13.4 llama.cpp b11803 or later, with matching CUDA runtime DLLs. b11803 downloads: https://github.com/ggml-org/llama.cpp/releases/tag/b11803.", "Download the linked flint-int4-q4_1.gguf into the folder containing llama-server.exe.", "Allow 28 GiB for inference plus operating-system headroom." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/RecursiveSI/flint-int4-gguf/blob/main/flint-int4-q4_1.gguf", "target_label": "RecursiveSI/flint-int4-gguf/flint-int4-q4_1.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11803. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11803\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch Recursive Flint 40B-A2.5B Q4_1", "target": "single-node", "instructions": "In PowerShell, open the folder containing llama-server.exe and the GGUF, then set $env:GGML_CUDA_MMQ_PREC = \"auto\" before launching.", "command": ".\\llama-server.exe -m flint-int4-q4_1.gguf `\n --no-mmproj `\n --host 127.0.0.1 `\n --port 57884 `\n --batch-size 4096 `\n --ubatch-size 2048 `\n --ctx-size 98304 `\n --parallel 1 `\n -ngl 999 `\n --n-predict 8192 `\n -fit off `\n --backend-sampling `\n --temp 0 `\n --top-p 1 `\n --top-k 1 `\n --min-p 0 `\n --presence-penalty 0 `\n --repeat-penalty 1 `\n -fa on `\n -ctk f16 `\n -ctv f16 `\n --jinja `\n --seed 1234 `\n --threads 12 `\n --threads-batch 12 `\n --metrics `\n --no-webui `\n --alias flint-q4-1", "port": 57884, "api_model": "flint-q4-1" }, "launch": null, "sampling": "Backend sampling: ON; temp 0; top-p 1; top-k 1; min-p 0; presence-penalty 0; repeat-penalty 1; seed 1234.", "notes": null, "clients": { "shell": "'{\"model\":\"flint-q4-1\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:57884/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:57884/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"flint-q4-1\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "recursivesi/flint/rtx-pro/windows/llama-cpp/q4-1", "hardware": "rtx-pro", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11803", "backend_min_version": "b11803", "min_vidmem_gb": 28, "precision": "q4-1", "variant": null, "status": "experimental", "prerequisites": [ "Windows x64 CUDA 13.4 llama.cpp b11803 or later, with matching CUDA runtime DLLs. b11803 downloads: https://github.com/ggml-org/llama.cpp/releases/tag/b11803.", "Download the linked flint-int4-q4_1.gguf into the folder containing llama-server.exe.", "Allow 28 GiB for inference plus operating-system headroom." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/RecursiveSI/flint-int4-gguf/blob/main/flint-int4-q4_1.gguf", "target_label": "RecursiveSI/flint-int4-gguf/flint-int4-q4_1.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11803. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11803\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-x64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch Recursive Flint 40B-A2.5B Q4_1", "target": "single-node", "instructions": "In PowerShell, open the folder containing llama-server.exe and the GGUF, then set $env:GGML_CUDA_MMQ_PREC = \"auto\" before launching.", "command": ".\\llama-server.exe -m flint-int4-q4_1.gguf `\n --no-mmproj `\n --host 127.0.0.1 `\n --port 49734 `\n --batch-size 2048 `\n --ubatch-size 2048 `\n --ctx-size 98304 `\n --parallel 1 `\n -ngl 999 `\n --n-predict 8192 `\n -fit off `\n --temp 0 `\n --top-p 1 `\n --top-k 1 `\n --min-p 0 `\n --presence-penalty 0 `\n --repeat-penalty 1 `\n -fa on `\n -ctk f16 `\n -ctv f16 `\n --jinja `\n --seed 1234 `\n --threads 12 `\n --threads-batch 12 `\n --metrics `\n --no-webui `\n --alias flint-q4-1", "port": 49734, "api_model": "flint-q4-1" }, "launch": null, "sampling": "Backend sampling: OFF; temp 0; top-p 1; top-k 1; min-p 0; presence-penalty 0; repeat-penalty 1; seed 1234.", "notes": null, "clients": { "shell": "'{\"model\":\"flint-q4-1\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:49734/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:49734/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"flint-q4-1\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "recursivesi/flint/rtx-spark/windows/llama-cpp/q4-1", "hardware": "rtx-spark", "platform": { "os": "windows", "execution": "native" }, "backend": "llama-cpp", "backend_version": "b11803", "backend_min_version": "b11803", "min_vidmem_gb": 28, "precision": "q4-1", "variant": null, "status": "experimental", "prerequisites": [ "Windows ARM64 CUDA 13.4 llama.cpp b11803 or later, with matching CUDA runtime DLLs. b11803 downloads: https://github.com/ggml-org/llama.cpp/releases/tag/b11803.", "Download the linked flint-int4-q4_1.gguf into the folder containing llama-server.exe.", "Allow 28 GiB for inference plus operating-system headroom." ], "limitations": [], "checkpoint": { "target": "https://huggingface.co/RecursiveSI/flint-int4-gguf/blob/main/flint-int4-q4_1.gguf", "target_label": "RecursiveSI/flint-int4-gguf/flint-int4-q4_1.gguf", "draft": null, "draft_label": null }, "security": null, "topology": null, "runtime": { "setup_label": "Download llama.cpp", "setup_instructions": "Pinned to b11803. Leaves you in the extracted folder with it added to PATH for this session. Optionally set $env:GITHUB_TOKEN = \"\" first to raise GitHub's API rate limit.", "setup_command": "&{\n cd $HOME\n $null = mkdir -Force llama-cuda\n cd .\\llama-cuda\n\n $headers = @{}\n if ($env:GITHUB_TOKEN) { $headers['Authorization'] = \"Bearer $env:GITHUB_TOKEN\" }\n try {\n $release = Invoke-RestMethod -Uri \"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/b11803\" -Headers $headers\n } catch {\n throw \"Failed to query the GitHub Releases API: $($_.Exception.Message)\"\n }\n\n $cudaAsset = $release.assets | Where-Object { $_.name -like \"cudart-llama-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n $llamaAsset = $release.assets | Where-Object { $_.name -like \"llama-*-bin-win-cuda-*-arm64.zip\" } | Select-Object -First 1\n if (-not $cudaAsset -or -not $llamaAsset) {\n throw \"No matching Windows CUDA release asset found for $($release.tag_name).\"\n }\n\n Invoke-WebRequest -Uri $cudaAsset.browser_download_url -OutFile $cudaAsset.name\n Invoke-WebRequest -Uri $llamaAsset.browser_download_url -OutFile $llamaAsset.name\n\n Expand-Archive $cudaAsset.name .\\llama -Force\n Expand-Archive $llamaAsset.name .\\llama -Force\n\n cd .\\llama\n $env:PATH = \"$PWD;$env:PATH\"\n}" }, "serve": { "label": "Launch Recursive Flint 40B-A2.5B Q4_1", "target": "single-node", "instructions": "In PowerShell, open the folder containing llama-server.exe and the GGUF, then set $env:GGML_CUDA_MMQ_PREC = \"auto\" before launching.", "command": ".\\llama-server.exe -m flint-int4-q4_1.gguf `\n --no-mmproj `\n --host 127.0.0.1 `\n --port 63099 `\n --batch-size 2048 `\n --ubatch-size 2048 `\n --ctx-size 98304 `\n --parallel 1 `\n -ngl 999 `\n --n-predict 8192 `\n -fit off `\n --backend-sampling `\n --temp 0 `\n --top-p 1 `\n --top-k 1 `\n --min-p 0 `\n --presence-penalty 0 `\n --repeat-penalty 1 `\n -fa on `\n -ctk f16 `\n -ctv f16 `\n --jinja `\n --seed 1234 `\n --threads 12 `\n --threads-batch 12 `\n --metrics `\n --no-webui `\n --alias flint-q4-1", "port": 63099, "api_model": "flint-q4-1" }, "launch": null, "sampling": "Backend sampling: ON; temp 0; top-p 1; top-k 1; min-p 0; presence-penalty 0; repeat-penalty 1; seed 1234.", "notes": null, "clients": { "shell": "'{\"model\":\"flint-q4-1\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}]}' | curl.exe 'http://127.0.0.1:63099/v1/chat/completions' -H 'Content-Type: application/json' --data-binary '@-'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:63099/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"flint-q4-1\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "z-ai/glm-5.2", "display_name": "GLM-5.2", "builder": "z-ai", "family": "GLM-5", "summary": "GLM-5.2 inference configurations.", "upstream": null, "modalities": [ "text" ], "recipes": [ { "id": "z-ai/glm-5.2/dgx-station-2gpu/linux/vllm/nvfp4", "hardware": "dgx-station-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "v0.31.0", "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4", "variant": null, "status": "experimental", "prerequisites": [ "Two DGX Stations with one GB300 each connected over one 400 Gb/s ConnectX-8 RoCE v2 rail.", "Docker access on both nodes with permission to select the GB300 by UUID and expose `/dev/infiniband`.", "NetworkManager, ethtool, show_gids, rsync, and passwordless SSH from the head to the worker RoCE address." ], "limitations": [ "Validated only on two DGX Stations with one GB300 per node over one 400 Gb/s ConnectX-8 RoCE v2 rail.", "The setup configures a persistent NetworkManager profile named `cx8-rail1` and requires jumbo frames with MTU 9000." ], "checkpoint": { "target": "https://huggingface.co/nvidia/GLM-5.2-NVFP4", "target_label": "nvidia/GLM-5.2-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and isolated model storage. The launch exposes `/dev/infiniband`, installs a pinned NCCL wheel when each container starts, and modifies persistent host networking with sudo; review these requirements before use. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× DGX Station with 1× GB300 each; one node rank per station; tensor parallel 2 over one 400 Gb/s ConnectX-8 RoCE v2 rail", "gpu_count": 2, "node_count": 2, "tensor_parallel_size": 2, "pipeline_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container (ARM64)", "image": "vllm/vllm-openai:v0.31.0-aarch64", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.31.0-aarch64", "setup_label": "Configure nodes and replicate checkpoint", "setup_target": "all-nodes", "setup_instructions": "Use Bash and replace every placeholder. Set `NTP_HOST` only when the NTP route needs correction. Start it on the worker with `NODE_ROLE=worker`; the worker waits up to six hours for the replicated checkpoint unless `MODEL_WAIT_TIMEOUT_SECONDS` is overridden. Then run it on the head with `NODE_ROLE=head`; the head downloads and replicates the checkpoint. Passwordless SSH to the worker RoCE address is required.", "setup_command": "set -euo pipefail\n\nHEAD_IP=\"\"\nWORKER_IP=\"\"\nRAIL_IFACE=\"\"\nRAIL_HCA=\"\"\n\nIMG=vllm/vllm-openai:v0.31.0-aarch64\nMODEL_REPO=nvidia/GLM-5.2-NVFP4\nMODEL_REV=aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa\nMODEL_HOST=\"\"\nCONTAINER=glm52-full\nPORT=8000\n\nNODE_ROLE=\"\"\ncase \"$NODE_ROLE\" in\n head)\n NODE_IP=$HEAD_IP\n PEER_IP=$WORKER_IP\n ;;\n worker)\n NODE_IP=$WORKER_IP\n PEER_IP=$HEAD_IP\n ;;\n *)\n echo \"NODE_ROLE must be head or worker\" >&2\n exit 1\n ;;\nesac\n\nif nmcli -t -f NAME connection show | grep -Fxq cx8-rail1; then\n sudo nmcli connection modify cx8-rail1 \\\n connection.interface-name \"$RAIL_IFACE\" connection.autoconnect yes \\\n ipv4.method manual ipv4.addresses \"$NODE_IP/24\" ipv4.never-default yes \\\n ipv6.method disabled 802-3-ethernet.mtu 9000\nelse\n sudo nmcli connection add type ethernet ifname \"$RAIL_IFACE\" con-name cx8-rail1 \\\n connection.autoconnect yes ipv4.method manual ipv4.addresses \"$NODE_IP/24\" \\\n ipv4.never-default yes ipv6.method disabled 802-3-ethernet.mtu 9000\nfi\nsudo nmcli connection up cx8-rail1\n\ntest \"$(cat /sys/class/net/$RAIL_IFACE/mtu)\" = 9000\nethtool \"$RAIL_IFACE\" | grep -q 'Speed: 400000Mb/s'\nethtool \"$RAIL_IFACE\" | grep -q 'Link detected: yes'\nping -I \"$RAIL_IFACE\" -M do -s 8972 -c 3 \"$PEER_IP\"\n\nGID_INDEX=$(/usr/sbin/show_gids 2>/dev/null | \\\n awk -v d=\"$RAIL_HCA\" -v ip=\"$NODE_IP\" \\\n '$1==d && $5==ip && $6==\"v2\" {gid=$3} END {if (gid!=\"\") print gid}')\ntest -n \"$GID_INDEX\"\necho \"$RAIL_HCA $NODE_IP GID_INDEX=$GID_INDEX\"\n\nfor c in port_rcv_errors port_xmit_discards symbol_error link_error_recovery link_downed; do\n test \"$(cat /sys/class/infiniband/$RAIL_HCA/ports/1/counters/$c)\" = 0\ndone\n\nNTP_HOST=${NTP_HOST:-}\nif [ -n \"$NTP_HOST\" ] && \\\n NTP_IP=$(getent ahostsv4 \"$NTP_HOST\" 2>/dev/null | awk 'NR==1 {print $1}') && \\\n [ -n \"$NTP_IP\" ] && \\\n ip route get \"$NTP_IP\" | head -1 | grep -q 'dev docker0'; then\n MGMT_IFACE=$(ip route show default | awk 'NR==1 {print $5}')\n MGMT_GW=$(ip route show default | awk 'NR==1 {print $3}')\n MGMT_CON=$(nmcli -g GENERAL.CONNECTION device show \"$MGMT_IFACE\")\n sudo ip route replace \"$NTP_IP/32\" via \"$MGMT_GW\" dev \"$MGMT_IFACE\"\n nmcli -g ipv4.routes connection show \"$MGMT_CON\" | grep -q \"$NTP_IP/32\" || \\\n sudo nmcli connection modify \"$MGMT_CON\" +ipv4.routes \"$NTP_IP/32 $MGMT_GW\"\n sudo systemctl restart systemd-timesyncd\nfi\ntest \"$(timedatectl show -p NTPSynchronized --value)\" = yes\n\nif [ \"$NODE_ROLE\" = head ]; then\n mkdir -p \"$MODEL_HOST\"\n docker run --rm --network host \\\n -v \"$MODEL_HOST:/model\" --entrypoint bash \"$IMG\" -lc \\\n \"exec hf download '$MODEL_REPO' --revision '$MODEL_REV' --local-dir /model\"\n\n ssh \"$WORKER_IP\" \"mkdir -p '$MODEL_HOST'\"\n rsync -aH --info=progress2 --exclude '.cache/' \\\n \"$MODEL_HOST/\" \"$WORKER_IP:$MODEL_HOST/\"\n\n LOCAL_MANIFEST=$(find \"$MODEL_HOST\" -path '*/.cache' -prune -o \\\n -type f -printf '%P %s\\n' | sort | sha256sum | awk '{print $1}')\n REMOTE_MANIFEST=$(ssh \"$WORKER_IP\" \"find '$MODEL_HOST' -path '*/.cache' -prune -o \\\n -type f -printf '%P %s\\\\n' | sort | sha256sum\" | awk '{print $1}')\n test \"$LOCAL_MANIFEST\" = \"$REMOTE_MANIFEST\"\n test \"$(find \"$MODEL_HOST\" -maxdepth 1 -name 'model-*-of-00047.safetensors' | wc -l)\" = 47\n echo \"MODEL_READY $LOCAL_MANIFEST\"\nelse\n mkdir -p \"$MODEL_HOST\"\n MODEL_WAIT_TIMEOUT_SECONDS=${MODEL_WAIT_TIMEOUT_SECONDS:-21600}\n test \"$MODEL_WAIT_TIMEOUT_SECONDS\" -gt 0\n MODEL_WAIT_DEADLINE=$((SECONDS + MODEL_WAIT_TIMEOUT_SECONDS))\n NEXT_PROGRESS=$SECONDS\n while :; do\n SHARD_COUNT=$(find \"$MODEL_HOST\" -maxdepth 1 \\\n -name 'model-*-of-00047.safetensors' | wc -l)\n if test \"$SHARD_COUNT\" = 47; then\n break\n fi\n if (( SECONDS >= MODEL_WAIT_DEADLINE )); then\n echo \"Timed out after ${MODEL_WAIT_TIMEOUT_SECONDS} seconds waiting for model replication; found $SHARD_COUNT/47 shards in $MODEL_HOST\" >&2\n find \"$MODEL_HOST\" -maxdepth 1 -name 'model-*.safetensors' \\\n -printf ' %f\\n' | sort >&2\n exit 1\n fi\n if (( SECONDS >= NEXT_PROGRESS )); then\n echo \"Waiting for model replication: $SHARD_COUNT/47 shards\"\n NEXT_PROGRESS=$((SECONDS + 60))\n fi\n sleep 10\n done\n test \"$(find \"$MODEL_HOST\" -maxdepth 1 -name 'model-*-of-00047.safetensors' | wc -l)\" = 47\n echo \"MODEL_READY\"\nfi" }, "serve": { "label": "Launch head", "target": "head", "instructions": "Run after the worker launch reports its container ID. Replace the RoCE, HCA, and model-path placeholders with the same values used during setup.", "command": "set -euo pipefail\n\nHEAD_IP=\"\"\nRAIL_IFACE=\"\"\nRAIL_HCA=\"\"\nIMG=vllm/vllm-openai:v0.31.0-aarch64\nSERVED=glm52\nMODEL_HOST=\"\"\nCONTAINER=glm52-full\nPORT=8000\n\nGPU_UUID=$(nvidia-smi --query-gpu=uuid,name --format=csv,noheader | \\\n awk -F, '$2 ~ /GB300/ {gsub(/ /,\"\",$1); print $1; exit}')\ntest -n \"$GPU_UUID\"\n\nRANK=0\nNODE_IP=$HEAD_IP\n\nGID_INDEX=$(/usr/sbin/show_gids 2>/dev/null | \\\n awk -v d=\"$RAIL_HCA\" -v ip=\"$NODE_IP\" \\\n '$1==d && $5==ip && $6==\"v2\" {gid=$3} END {if (gid!=\"\") print gid}')\ntest -n \"$GID_INDEX\"\n\nSPEC_CONFIG='{\"method\":\"mtp\",\"num_speculative_tokens\":4}'\nKERNEL_CONFIG='{\"enable_flashinfer_autotune\":false,\"moe_backend\":\"flashinfer_cutlass\"}'\nCOMPILE_CONFIG='{\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"cudagraph_capture_sizes\":[5,10,20,40,80],\"pass_config\":{\"fuse_allreduce_rms\":false}}'\n\ndocker rm -f \"$CONTAINER\" >/dev/null 2>&1 || true\ndocker run -d --name \"$CONTAINER\" \\\n --gpus device=\"$GPU_UUID\" --network host --ipc host --shm-size 32g \\\n --device /dev/infiniband --cap-add IPC_LOCK \\\n --ulimit memlock=-1 --ulimit stack=67108864 \\\n -v \"$MODEL_HOST:/model:ro\" \\\n -e RANK=\"$RANK\" \\\n -e MASTER_ADDR=\"$HEAD_IP\" -e SERVED=\"$SERVED\" -e PORT=\"$PORT\" \\\n -e SPEC_CONFIG=\"$SPEC_CONFIG\" -e KERNEL_CONFIG=\"$KERNEL_CONFIG\" \\\n -e COMPILE_CONFIG=\"$COMPILE_CONFIG\" \\\n -e NCCL_CUMEM_ENABLE=1 -e NCCL_IB_HCA=\"$RAIL_HCA\" \\\n -e NCCL_IB_GID_INDEX=\"$GID_INDEX\" \\\n -e NCCL_SOCKET_IFNAME=\"$RAIL_IFACE\" -e GLOO_SOCKET_IFNAME=\"$RAIL_IFACE\" \\\n -e NCCL_NET_GDR_LEVEL=5 -e NCCL_DEBUG=INFO -e NCCL_DEBUG_SUBSYS=INIT,NET \\\n -e NCCL_DEBUG_FILE='/tmp/nccl_%h_%p.log' -e VLLM_HOST_IP=\"$NODE_IP\" \\\n -e VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 \\\n -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \\\n -e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 -e FLASHINFER_DISABLE_VERSION_CHECK=1 \\\n --entrypoint bash \"$IMG\" -lc '\n set -euo pipefail\n pip install -q --no-deps --force-reinstall nvidia-nccl-cu13==2.30.7\n export VLLM_NCCL_SO_PATH=$(python3 -c \"import nvidia.nccl,os; print(os.path.join(list(nvidia.nccl.__path__)[0], \\\"lib\\\", \\\"libnccl.so.2\\\"))\")\n test -f \"$VLLM_NCCL_SO_PATH\"\n exec vllm serve /model --served-model-name \"$SERVED\" \\\n --host 0.0.0.0 --port \"$PORT\" \\\n --nnodes 2 --node-rank \"$RANK\" --master-addr \"$MASTER_ADDR\" --master-port 29501 \\\n --tensor-parallel-size 2 \\\n --speculative-config \"$SPEC_CONFIG\" \\\n --max-model-len 350000 --block-size 128 --max-num-seqs 16 \\\n --trust-remote-code --disable-custom-all-reduce \\\n --kv-cache-dtype fp8 --enable-prefix-caching \\\n --gpu-memory-utilization 0.98 --cpu-offload-gb 0 \\\n --kernel-config \"$KERNEL_CONFIG\" --compilation-config \"$COMPILE_CONFIG\" \\\n --enable-auto-tool-choice --tool-call-parser glm47 --reasoning-parser glm47\n '", "port": 8000, "api_model": "glm52", "additional_commands": [ { "id": "worker", "label": "Launch worker", "target": "workers", "instructions": "Run first on the worker. Replace the RoCE, HCA, and model-path placeholders with the same values used during setup.", "command": "set -euo pipefail\n\nHEAD_IP=\"\"\nWORKER_IP=\"\"\nRAIL_IFACE=\"\"\nRAIL_HCA=\"\"\nIMG=vllm/vllm-openai:v0.31.0-aarch64\nSERVED=glm52\nMODEL_HOST=\"\"\nCONTAINER=glm52-full\nPORT=8000\n\nGPU_UUID=$(nvidia-smi --query-gpu=uuid,name --format=csv,noheader | \\\n awk -F, '$2 ~ /GB300/ {gsub(/ /,\"\",$1); print $1; exit}')\ntest -n \"$GPU_UUID\"\n\nRANK=1\nNODE_IP=$WORKER_IP\n\nGID_INDEX=$(/usr/sbin/show_gids 2>/dev/null | \\\n awk -v d=\"$RAIL_HCA\" -v ip=\"$NODE_IP\" \\\n '$1==d && $5==ip && $6==\"v2\" {gid=$3} END {if (gid!=\"\") print gid}')\ntest -n \"$GID_INDEX\"\n\nSPEC_CONFIG='{\"method\":\"mtp\",\"num_speculative_tokens\":4}'\nKERNEL_CONFIG='{\"enable_flashinfer_autotune\":false,\"moe_backend\":\"flashinfer_cutlass\"}'\nCOMPILE_CONFIG='{\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"cudagraph_capture_sizes\":[5,10,20,40,80],\"pass_config\":{\"fuse_allreduce_rms\":false}}'\n\ndocker rm -f \"$CONTAINER\" >/dev/null 2>&1 || true\ndocker run -d --name \"$CONTAINER\" \\\n --gpus device=\"$GPU_UUID\" --network host --ipc host --shm-size 32g \\\n --device /dev/infiniband --cap-add IPC_LOCK \\\n --ulimit memlock=-1 --ulimit stack=67108864 \\\n -v \"$MODEL_HOST:/model:ro\" \\\n -e RANK=\"$RANK\" \\\n -e MASTER_ADDR=\"$HEAD_IP\" -e SERVED=\"$SERVED\" -e PORT=\"$PORT\" \\\n -e SPEC_CONFIG=\"$SPEC_CONFIG\" -e KERNEL_CONFIG=\"$KERNEL_CONFIG\" \\\n -e COMPILE_CONFIG=\"$COMPILE_CONFIG\" \\\n -e NCCL_CUMEM_ENABLE=1 -e NCCL_IB_HCA=\"$RAIL_HCA\" \\\n -e NCCL_IB_GID_INDEX=\"$GID_INDEX\" \\\n -e NCCL_SOCKET_IFNAME=\"$RAIL_IFACE\" -e GLOO_SOCKET_IFNAME=\"$RAIL_IFACE\" \\\n -e NCCL_NET_GDR_LEVEL=5 -e NCCL_DEBUG=INFO -e NCCL_DEBUG_SUBSYS=INIT,NET \\\n -e NCCL_DEBUG_FILE='/tmp/nccl_%h_%p.log' -e VLLM_HOST_IP=\"$NODE_IP\" \\\n -e VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 \\\n -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \\\n -e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 -e FLASHINFER_DISABLE_VERSION_CHECK=1 \\\n --entrypoint bash \"$IMG\" -lc '\n set -euo pipefail\n pip install -q --no-deps --force-reinstall nvidia-nccl-cu13==2.30.7\n export VLLM_NCCL_SO_PATH=$(python3 -c \"import nvidia.nccl,os; print(os.path.join(list(nvidia.nccl.__path__)[0], \\\"lib\\\", \\\"libnccl.so.2\\\"))\")\n test -f \"$VLLM_NCCL_SO_PATH\"\n exec vllm serve /model --served-model-name \"$SERVED\" \\\n --host 0.0.0.0 --port \"$PORT\" \\\n --nnodes 2 --node-rank \"$RANK\" --master-addr \"$MASTER_ADDR\" --master-port 29501 \\\n --tensor-parallel-size 2 \\\n --speculative-config \"$SPEC_CONFIG\" \\\n --max-model-len 350000 --block-size 128 --max-num-seqs 16 \\\n --trust-remote-code --disable-custom-all-reduce \\\n --kv-cache-dtype fp8 --enable-prefix-caching \\\n --gpu-memory-utilization 0.98 --cpu-offload-gb 0 \\\n --kernel-config \"$KERNEL_CONFIG\" --compilation-config \"$COMPILE_CONFIG\" \\\n --enable-auto-tool-choice --tool-call-parser glm47 --reasoning-parser glm47 \\\n --headless\n '" } ] }, "launch": null, "sampling": null, "notes": "Start the worker before the head. The setup requires the same model path on both nodes. Verify both NCCL logs show `NET/IB` or `GDRDMA`, contain the selected HCA, show no `NET/Socket` fallback, and contain no `NCCL WARN` or `NCCL ERROR` entries.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"glm52\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"glm52\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "z-ai/glm-5.3", "display_name": "GLM-5.3", "builder": "z-ai", "family": "GLM-5", "summary": "GLM-5.3 inference configurations.", "upstream": "https://huggingface.co/nvidia/GLM-5.3-NVFP4", "modalities": [ "text" ], "recipes": [ { "id": "z-ai/glm-5.3/dgx-station-2gpu/linux/vllm/nvfp4", "hardware": "dgx-station-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "v0.30.0", "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4", "variant": null, "status": "experimental", "prerequisites": [ "Two DGX Stations with one GB300 each connected over one 400 Gb/s ConnectX-8 RoCE v2 rail.", "Docker access on both nodes with permission to select the GB300 by UUID and expose `/dev/infiniband`.", "NetworkManager, ethtool, show_gids, rsync, and passwordless SSH from the head to the worker RoCE address." ], "limitations": [ "Validated only on two DGX Stations with one GB300 per node over ConnectX-8 RoCE v2.", "Context is capped at 98,304 tokens with `--gpu-memory-utilization 0.93`; this image allocates the sparse-MLA prefill workspace after KV sizing, and higher utilization or longer context ran out of memory on the first long prompt.", "The setup configures a persistent NetworkManager profile named `cx8-rail1` and requires jumbo frames with MTU 9000." ], "checkpoint": { "target": "https://huggingface.co/nvidia/GLM-5.3-NVFP4", "target_label": "nvidia/GLM-5.3-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and isolated model storage. The launch exposes `/dev/infiniband`, installs a pinned NCCL wheel when each container starts, and modifies persistent host networking with sudo; review these requirements before use. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× DGX Station with 1× GB300 each; one node rank per station; tensor parallel 2 over one 400 Gb/s ConnectX-8 RoCE v2 rail", "gpu_count": 2, "node_count": 2, "tensor_parallel_size": 2, "pipeline_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container (ARM64)", "image": "vllm/vllm-openai:v0.30.0-aarch64", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.30.0-aarch64", "setup_label": "Configure nodes and replicate checkpoint", "setup_target": "all-nodes", "setup_instructions": "Use Bash and replace every placeholder. Set `NTP_HOST` only when the NTP route needs correction. Start it on the worker with `NODE_ROLE=worker`; the worker waits up to six hours for the replicated checkpoint unless `MODEL_WAIT_TIMEOUT_SECONDS` is overridden. Then run it on the head with `NODE_ROLE=head`; the head downloads and replicates the checkpoint. Passwordless SSH to the worker RoCE address is required.", "setup_command": "set -euo pipefail\n\nHEAD_IP=\"\"\nWORKER_IP=\"\"\nRAIL_IFACE=\"\"\nRAIL_HCA=\"\"\n\nIMG=vllm/vllm-openai:v0.30.0-aarch64\nMODEL_REPO=nvidia/GLM-5.3-NVFP4\nMODEL_REV=e3b8f9f2fad29727dfa95e766c5cac18ea1414bf\nMODEL_HOST=\"\"\nCONTAINER=glm53-full\nPORT=8000\n\nNODE_ROLE=\"\"\ncase \"$NODE_ROLE\" in\n head)\n NODE_IP=$HEAD_IP\n PEER_IP=$WORKER_IP\n ;;\n worker)\n NODE_IP=$WORKER_IP\n PEER_IP=$HEAD_IP\n ;;\n *)\n echo \"NODE_ROLE must be head or worker\" >&2\n exit 1\n ;;\nesac\n\nif nmcli -t -f NAME connection show | grep -Fxq cx8-rail1; then\n sudo nmcli connection modify cx8-rail1 \\\n connection.interface-name \"$RAIL_IFACE\" connection.autoconnect yes \\\n ipv4.method manual ipv4.addresses \"$NODE_IP/24\" ipv4.never-default yes \\\n ipv6.method disabled 802-3-ethernet.mtu 9000\nelse\n sudo nmcli connection add type ethernet ifname \"$RAIL_IFACE\" con-name cx8-rail1 \\\n connection.autoconnect yes ipv4.method manual ipv4.addresses \"$NODE_IP/24\" \\\n ipv4.never-default yes ipv6.method disabled 802-3-ethernet.mtu 9000\nfi\nsudo nmcli connection up cx8-rail1\n\ntest \"$(cat /sys/class/net/$RAIL_IFACE/mtu)\" = 9000\nethtool \"$RAIL_IFACE\" | grep -q 'Speed: 400000Mb/s'\nethtool \"$RAIL_IFACE\" | grep -q 'Link detected: yes'\nping -I \"$RAIL_IFACE\" -M do -s 8972 -c 3 \"$PEER_IP\"\n\nGID_INDEX=$(/usr/sbin/show_gids 2>/dev/null | \\\n awk -v d=\"$RAIL_HCA\" -v ip=\"$NODE_IP\" \\\n '$1==d && $5==ip && $6==\"v2\" {gid=$3} END {if (gid!=\"\") print gid}')\ntest -n \"$GID_INDEX\"\necho \"$RAIL_HCA $NODE_IP GID_INDEX=$GID_INDEX\"\n\nfor c in port_rcv_errors port_xmit_discards symbol_error link_error_recovery link_downed; do\n test \"$(cat /sys/class/infiniband/$RAIL_HCA/ports/1/counters/$c)\" = 0\ndone\n\nNTP_HOST=${NTP_HOST:-}\nif [ -n \"$NTP_HOST\" ] && \\\n NTP_IP=$(getent ahostsv4 \"$NTP_HOST\" 2>/dev/null | awk 'NR==1 {print $1}') && \\\n [ -n \"$NTP_IP\" ] && \\\n ip route get \"$NTP_IP\" | head -1 | grep -q 'dev docker0'; then\n MGMT_IFACE=$(ip route show default | awk 'NR==1 {print $5}')\n MGMT_GW=$(ip route show default | awk 'NR==1 {print $3}')\n MGMT_CON=$(nmcli -g GENERAL.CONNECTION device show \"$MGMT_IFACE\")\n sudo ip route replace \"$NTP_IP/32\" via \"$MGMT_GW\" dev \"$MGMT_IFACE\"\n nmcli -g ipv4.routes connection show \"$MGMT_CON\" | grep -q \"$NTP_IP/32\" || \\\n sudo nmcli connection modify \"$MGMT_CON\" +ipv4.routes \"$NTP_IP/32 $MGMT_GW\"\n sudo systemctl restart systemd-timesyncd\nfi\ntest \"$(timedatectl show -p NTPSynchronized --value)\" = yes\n\nif [ \"$NODE_ROLE\" = head ]; then\n mkdir -p \"$MODEL_HOST\"\n docker run --rm --network host \\\n -v \"$MODEL_HOST:/model\" --entrypoint bash \"$IMG\" -lc \\\n \"exec hf download '$MODEL_REPO' --revision '$MODEL_REV' --local-dir /model\"\n\n ssh \"$WORKER_IP\" \"mkdir -p '$MODEL_HOST'\"\n rsync -aH --info=progress2 --exclude '.cache/' \\\n \"$MODEL_HOST/\" \"$WORKER_IP:$MODEL_HOST/\"\n\n LOCAL_MANIFEST=$(find \"$MODEL_HOST\" -path '*/.cache' -prune -o \\\n -type f -printf '%P %s\\n' | sort | sha256sum | awk '{print $1}')\n REMOTE_MANIFEST=$(ssh \"$WORKER_IP\" \"find '$MODEL_HOST' -path '*/.cache' -prune -o \\\n -type f -printf '%P %s\\\\n' | sort | sha256sum\" | awk '{print $1}')\n test \"$LOCAL_MANIFEST\" = \"$REMOTE_MANIFEST\"\n test \"$(find \"$MODEL_HOST\" -maxdepth 1 -name 'model-*-of-00050.safetensors' | wc -l)\" = 50\n python3 -c \"import json; c=json.load(open('$MODEL_HOST/config.json')); assert c['architectures']==['GlmMoeDsaForCausalLM']; assert c['quantization_config']['quant_algo']=='MIXED_PRECISION'; print('CONFIG_OK')\"\n echo \"MODEL_READY $LOCAL_MANIFEST\"\nelse\n mkdir -p \"$MODEL_HOST\"\n MODEL_WAIT_TIMEOUT_SECONDS=${MODEL_WAIT_TIMEOUT_SECONDS:-21600}\n test \"$MODEL_WAIT_TIMEOUT_SECONDS\" -gt 0\n MODEL_WAIT_DEADLINE=$((SECONDS + MODEL_WAIT_TIMEOUT_SECONDS))\n NEXT_PROGRESS=$SECONDS\n while :; do\n SHARD_COUNT=$(find \"$MODEL_HOST\" -maxdepth 1 \\\n -name 'model-*-of-00050.safetensors' | wc -l)\n if test \"$SHARD_COUNT\" = 50; then\n break\n fi\n if (( SECONDS >= MODEL_WAIT_DEADLINE )); then\n echo \"Timed out after ${MODEL_WAIT_TIMEOUT_SECONDS} seconds waiting for model replication; found $SHARD_COUNT/50 shards in $MODEL_HOST\" >&2\n find \"$MODEL_HOST\" -maxdepth 1 -name 'model-*.safetensors' \\\n -printf ' %f\\n' | sort >&2\n exit 1\n fi\n if (( SECONDS >= NEXT_PROGRESS )); then\n echo \"Waiting for model replication: $SHARD_COUNT/50 shards\"\n NEXT_PROGRESS=$((SECONDS + 60))\n fi\n sleep 10\n done\n test \"$(find \"$MODEL_HOST\" -maxdepth 1 -name 'model-*-of-00050.safetensors' | wc -l)\" = 50\n echo \"MODEL_READY\"\nfi" }, "serve": { "label": "Launch head", "target": "head", "instructions": "Run after the worker launch reports its container ID. Replace the RoCE, HCA, and model-path placeholders with the same values used during setup.", "command": "set -euo pipefail\n\nHEAD_IP=\"\"\nRAIL_IFACE=\"\"\nRAIL_HCA=\"\"\nIMG=vllm/vllm-openai:v0.30.0-aarch64\nMODEL_REPO=nvidia/GLM-5.3-NVFP4\nMODEL_HOST=\"\"\nCONTAINER=glm53-full\nPORT=8000\n\nGPU_UUID=$(nvidia-smi --query-gpu=uuid,name --format=csv,noheader | \\\n awk -F, '$2 ~ /GB300/ {gsub(/ /,\"\",$1); print $1; exit}')\ntest -n \"$GPU_UUID\"\n\nRANK=0\nNODE_IP=$HEAD_IP\n\nGID_INDEX=$(/usr/sbin/show_gids 2>/dev/null | \\\n awk -v d=\"$RAIL_HCA\" -v ip=\"$NODE_IP\" \\\n '$1==d && $5==ip && $6==\"v2\" {gid=$3} END {if (gid!=\"\") print gid}')\ntest -n \"$GID_INDEX\"\n\nSPEC_CONFIG='{\"method\":\"mtp\",\"num_speculative_tokens\":4}'\nKERNEL_CONFIG='{\"enable_flashinfer_autotune\":false,\"moe_backend\":\"flashinfer_cutlass\"}'\nCOMPILE_CONFIG='{\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"cudagraph_capture_sizes\":[5,10,20,40,80],\"pass_config\":{\"fuse_allreduce_rms\":false}}'\n\ndocker rm -f \"$CONTAINER\" >/dev/null 2>&1 || true\ndocker run -d --name \"$CONTAINER\" \\\n --gpus device=\"$GPU_UUID\" --network host --ipc host --shm-size 32g \\\n --device /dev/infiniband --cap-add IPC_LOCK \\\n --ulimit memlock=-1 --ulimit stack=67108864 \\\n -v \"$MODEL_HOST:/model:ro\" \\\n -e RANK=\"$RANK\" \\\n -e MASTER_ADDR=\"$HEAD_IP\" -e MODEL_REPO=\"$MODEL_REPO\" -e PORT=\"$PORT\" \\\n -e SPEC_CONFIG=\"$SPEC_CONFIG\" -e KERNEL_CONFIG=\"$KERNEL_CONFIG\" \\\n -e COMPILE_CONFIG=\"$COMPILE_CONFIG\" \\\n -e NCCL_CUMEM_ENABLE=1 -e NCCL_IB_HCA=\"$RAIL_HCA\" \\\n -e NCCL_IB_GID_INDEX=\"$GID_INDEX\" \\\n -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 \\\n -e VLLM_ALLREDUCE_USE_FLASHINFER=0 -e VLLM_ALLREDUCE_USE_SYMM_MEM=0 \\\n -e NCCL_SOCKET_IFNAME=\"$RAIL_IFACE\" -e GLOO_SOCKET_IFNAME=\"$RAIL_IFACE\" \\\n -e NCCL_NET_GDR_LEVEL=5 -e NCCL_DEBUG=INFO -e NCCL_DEBUG_SUBSYS=INIT,NET \\\n -e NCCL_DEBUG_FILE='/tmp/nccl_%h_%p.log' -e VLLM_HOST_IP=\"$NODE_IP\" \\\n -e VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 \\\n -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \\\n -e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 -e FLASHINFER_DISABLE_VERSION_CHECK=1 \\\n --entrypoint bash \"$IMG\" -lc '\n set -euo pipefail\n pip install -q --no-deps --force-reinstall nvidia-nccl-cu13==2.30.7\n export VLLM_NCCL_SO_PATH=$(python3 -c \"import nvidia.nccl,os; print(os.path.join(list(nvidia.nccl.__path__)[0], \\\"lib\\\", \\\"libnccl.so.2\\\"))\")\n test -f \"$VLLM_NCCL_SO_PATH\"\n python3 -c \"import transformers; from transformers import GlmMoeDsaConfig; print(\\\"transformers\\\", transformers.__version__)\"\n exec vllm serve /model --served-model-name \"$MODEL_REPO\" \\\n --host 0.0.0.0 --port \"$PORT\" \\\n --nnodes 2 --node-rank \"$RANK\" --master-addr \"$MASTER_ADDR\" --master-port 29501 \\\n --tensor-parallel-size 2 \\\n --speculative-config \"$SPEC_CONFIG\" \\\n --max-model-len 98304 --max-num-batched-tokens 8192 --block-size 128 --max-num-seqs 16 \\\n --trust-remote-code --disable-custom-all-reduce \\\n --kv-cache-dtype fp8 --enable-prefix-caching \\\n --gpu-memory-utilization 0.93 --cpu-offload-gb 0 \\\n --kernel-config \"$KERNEL_CONFIG\" --compilation-config \"$COMPILE_CONFIG\" \\\n --distributed-timeout-seconds 1800 \\\n --enable-auto-tool-choice --tool-call-parser glm47 --reasoning-parser glm47\n '", "port": 8000, "api_model": "nvidia/GLM-5.3-NVFP4", "additional_commands": [ { "id": "worker", "label": "Launch worker", "target": "workers", "instructions": "Run first on the worker. Replace the RoCE, HCA, and model-path placeholders with the same values used during setup.", "command": "set -euo pipefail\n\nHEAD_IP=\"\"\nWORKER_IP=\"\"\nRAIL_IFACE=\"\"\nRAIL_HCA=\"\"\nIMG=vllm/vllm-openai:v0.30.0-aarch64\nMODEL_REPO=nvidia/GLM-5.3-NVFP4\nMODEL_HOST=\"\"\nCONTAINER=glm53-full\nPORT=8000\n\nGPU_UUID=$(nvidia-smi --query-gpu=uuid,name --format=csv,noheader | \\\n awk -F, '$2 ~ /GB300/ {gsub(/ /,\"\",$1); print $1; exit}')\ntest -n \"$GPU_UUID\"\n\nRANK=1\nNODE_IP=$WORKER_IP\n\nGID_INDEX=$(/usr/sbin/show_gids 2>/dev/null | \\\n awk -v d=\"$RAIL_HCA\" -v ip=\"$NODE_IP\" \\\n '$1==d && $5==ip && $6==\"v2\" {gid=$3} END {if (gid!=\"\") print gid}')\ntest -n \"$GID_INDEX\"\n\nSPEC_CONFIG='{\"method\":\"mtp\",\"num_speculative_tokens\":4}'\nKERNEL_CONFIG='{\"enable_flashinfer_autotune\":false,\"moe_backend\":\"flashinfer_cutlass\"}'\nCOMPILE_CONFIG='{\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"cudagraph_capture_sizes\":[5,10,20,40,80],\"pass_config\":{\"fuse_allreduce_rms\":false}}'\n\ndocker rm -f \"$CONTAINER\" >/dev/null 2>&1 || true\ndocker run -d --name \"$CONTAINER\" \\\n --gpus device=\"$GPU_UUID\" --network host --ipc host --shm-size 32g \\\n --device /dev/infiniband --cap-add IPC_LOCK \\\n --ulimit memlock=-1 --ulimit stack=67108864 \\\n -v \"$MODEL_HOST:/model:ro\" \\\n -e RANK=\"$RANK\" \\\n -e MASTER_ADDR=\"$HEAD_IP\" -e MODEL_REPO=\"$MODEL_REPO\" -e PORT=\"$PORT\" \\\n -e SPEC_CONFIG=\"$SPEC_CONFIG\" -e KERNEL_CONFIG=\"$KERNEL_CONFIG\" \\\n -e COMPILE_CONFIG=\"$COMPILE_CONFIG\" \\\n -e NCCL_CUMEM_ENABLE=1 -e NCCL_IB_HCA=\"$RAIL_HCA\" \\\n -e NCCL_IB_GID_INDEX=\"$GID_INDEX\" \\\n -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 \\\n -e VLLM_ALLREDUCE_USE_FLASHINFER=0 -e VLLM_ALLREDUCE_USE_SYMM_MEM=0 \\\n -e NCCL_SOCKET_IFNAME=\"$RAIL_IFACE\" -e GLOO_SOCKET_IFNAME=\"$RAIL_IFACE\" \\\n -e NCCL_NET_GDR_LEVEL=5 -e NCCL_DEBUG=INFO -e NCCL_DEBUG_SUBSYS=INIT,NET \\\n -e NCCL_DEBUG_FILE='/tmp/nccl_%h_%p.log' -e VLLM_HOST_IP=\"$NODE_IP\" \\\n -e VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 \\\n -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \\\n -e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 -e FLASHINFER_DISABLE_VERSION_CHECK=1 \\\n --entrypoint bash \"$IMG\" -lc '\n set -euo pipefail\n pip install -q --no-deps --force-reinstall nvidia-nccl-cu13==2.30.7\n export VLLM_NCCL_SO_PATH=$(python3 -c \"import nvidia.nccl,os; print(os.path.join(list(nvidia.nccl.__path__)[0], \\\"lib\\\", \\\"libnccl.so.2\\\"))\")\n test -f \"$VLLM_NCCL_SO_PATH\"\n python3 -c \"import transformers; from transformers import GlmMoeDsaConfig; print(\\\"transformers\\\", transformers.__version__)\"\n exec vllm serve /model --served-model-name \"$MODEL_REPO\" \\\n --host 0.0.0.0 --port \"$PORT\" \\\n --nnodes 2 --node-rank \"$RANK\" --master-addr \"$MASTER_ADDR\" --master-port 29501 \\\n --tensor-parallel-size 2 \\\n --speculative-config \"$SPEC_CONFIG\" \\\n --max-model-len 98304 --max-num-batched-tokens 8192 --block-size 128 --max-num-seqs 16 \\\n --trust-remote-code --disable-custom-all-reduce \\\n --kv-cache-dtype fp8 --enable-prefix-caching \\\n --gpu-memory-utilization 0.93 --cpu-offload-gb 0 \\\n --kernel-config \"$KERNEL_CONFIG\" --compilation-config \"$COMPILE_CONFIG\" \\\n --distributed-timeout-seconds 1800 \\\n --enable-auto-tool-choice --tool-call-parser glm47 --reasoning-parser glm47 \\\n --headless\n '" } ] }, "launch": null, "sampling": null, "notes": "Start the worker before the head. The setup requires the same model path on both nodes. Verify both NCCL logs show `NET/IB` or `GDRDMA`, contain the selected HCA, show no `NET/Socket` fallback, and contain no `NCCL WARN` or `NCCL ERROR` entries. GLM-5.3 always thinks; clients set the budget with `\"chat_template_kwargs\": {\"reasoning_effort\": \"low\" | \"high\" | \"max\"}` (default `max`).", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/GLM-5.3-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/GLM-5.3-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "z-ai/glm-5.3/dgx-station/linux/vllm/nvfp4", "hardware": "dgx-station", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "v0.30.0", "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4", "variant": null, "status": "optimized", "prerequisites": [ "One DGX Station with one GB300, at least 500 GiB of free disk space, at least 250 GiB of free host memory, and no competing workload.", "Docker access with permission to run a privileged container and select the GB300 by UUID." ], "limitations": [ "Validated only on one DGX Station with one GB300 using the pinned ARM64 vLLM image and immutable checkpoint revision shown below.", "The checkpoint does not fit in one GB300; about 224 GiB of routed-expert weights stay in host memory and are read over C2C, so decode is bandwidth-bound (about 30 tokens/s for one user).", "Context is capped at 131,072 tokens.", "The container runs privileged so it can clear the host page cache before vLLM starts." ], "checkpoint": { "target": "https://huggingface.co/nvidia/GLM-5.3-NVFP4", "target_label": "nvidia/GLM-5.3-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. The container runs privileged and writes to `/proc/sys/vm/drop_caches`; review this requirement before use. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Station with 1× GB300; tensor parallel 1; UVA offload of routed-expert weights", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container (ARM64)", "image": "vllm/vllm-openai:v0.30.0-aarch64", "image_url": "https://hub.docker.com/layers/vllm/vllm-openai/v0.30.0-aarch64", "setup_label": "Download checkpoint", "setup_target": "single-node", "setup_instructions": "Use Bash and replace the host-path placeholders. The checkpoint download is pinned to the validated immutable revision.", "setup_command": "set -euo pipefail\n\nIMG=vllm/vllm-openai:v0.30.0-aarch64\nMODEL_REPO=nvidia/GLM-5.3-NVFP4\nMODEL_REV=e3b8f9f2fad29727dfa95e766c5cac18ea1414bf\nMODEL_HOST=\"\"\nCACHE_HOST=\"\"\n\nmkdir -p \"$MODEL_HOST\" \"$CACHE_HOST\"\ndocker run --rm --network host \\\n -v \"$MODEL_HOST:/model\" --entrypoint bash \"$IMG\" -lc \\\n \"exec hf download '$MODEL_REPO' --revision '$MODEL_REV' --local-dir /model\"\n\ntest \"$(find \"$MODEL_HOST\" -maxdepth 1 \\\n -name 'model-*-of-00050.safetensors' | wc -l)\" = 50" }, "serve": { "label": "Start TP=1 with UVA expert offload and MTP", "target": "single-node", "instructions": "Use Bash, replace the host-path placeholders with the same values used during setup, and run on an otherwise idle station. Startup takes about 6 minutes; the first requests compile kernels and run slowly.", "command": "set -euo pipefail\n\nIMG=vllm/vllm-openai:v0.30.0-aarch64\nMODEL_REPO=nvidia/GLM-5.3-NVFP4\nMODEL_HOST=\"\"\nCACHE_HOST=\"\"\nCONTAINER=glm53-single\nPORT=8000\n\nGPU_UUID=$(nvidia-smi --query-gpu=uuid,name --format=csv,noheader | \\\n awk -F, '$2 ~ /GB300/ {gsub(/ /,\"\",$1); print $1; exit}')\ntest -n \"$GPU_UUID\"\n\ndocker rm -f \"$CONTAINER\" >/dev/null 2>&1 || true\ndocker run -d --name \"$CONTAINER\" \\\n --gpus device=\"$GPU_UUID\" --network host --ipc host --shm-size 32g \\\n --privileged --ulimit memlock=-1 --ulimit stack=67108864 \\\n -v \"$MODEL_HOST:/model:ro\" -v \"$CACHE_HOST:/root/.cache\" \\\n -e MODEL_REPO=\"$MODEL_REPO\" -e PORT=\"$PORT\" \\\n -e SPEC_CONFIG='{\"method\":\"mtp\",\"num_speculative_tokens\":4}' \\\n -e KERNEL_CONFIG='{\"enable_flashinfer_autotune\":false,\"moe_backend\":\"flashinfer_cutlass\"}' \\\n -e COMPILE_CONFIG='{\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"cudagraph_capture_sizes\":[5,10,20,40,80]}' \\\n -e VLLM_USE_V2_MODEL_RUNNER=1 \\\n -e VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY=1 \\\n -e VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600 \\\n -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 \\\n --entrypoint bash \"$IMG\" -lc '\n set -euo pipefail\n sync\n echo 3 > /proc/sys/vm/drop_caches\n exec vllm serve /model \\\n --served-model-name \"$MODEL_REPO\" --host 0.0.0.0 --port \"$PORT\" \\\n --tensor-parallel-size 1 --trust-remote-code \\\n --gpu-memory-utilization 0.95 --kv-cache-dtype fp8_e4m3 \\\n --max-model-len 131072 --max-num-batched-tokens 8192 --max-num-seqs 16 \\\n --enable-prefix-caching \\\n --offload-backend uva --cpu-offload-gb 222 \\\n --cpu-offload-params w13_weight w2_weight \\\n --speculative-config \"$SPEC_CONFIG\" --max-cudagraph-capture-size 80 \\\n --reasoning-parser glm47 --tool-call-parser glm47 --enable-auto-tool-choice \\\n --kernel-config \"$KERNEL_CONFIG\" --compilation-config \"$COMPILE_CONFIG\"\n '", "port": 8000, "api_model": "nvidia/GLM-5.3-NVFP4" }, "launch": null, "sampling": null, "notes": "Keep `cudagraph_mode: FULL_DECODE_ONLY` and the `flashinfer_cutlass` MoE backend: with the vLLM defaults (piecewise graphs, autotuned MoE kernel, V1 model runner) this checkpoint produced repeated-comma output on this image. GLM-5.3 always thinks; clients set the budget with `\"chat_template_kwargs\": {\"reasoning_effort\": \"low\" | \"high\" | \"max\"}`.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/GLM-5.3-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/GLM-5.3-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] }, { "id": "z-ai/glm-5.3-flash", "display_name": "GLM-5.3-Flash", "builder": "z-ai", "family": "GLM-5", "summary": "GLM-5.3-Flash inference configurations.", "upstream": null, "modalities": [ "text" ], "recipes": [ { "id": "z-ai/glm-5.3-flash/dgx-spark-2gpu/linux/vllm/fp4-local", "hardware": "dgx-spark-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21830+g8244d1c44.d20261015", "backend_min_version": null, "min_vidmem_gb": 128, "precision": "fp4-local", "variant": null, "status": "optimized", "prerequisites": [ "Two DGX Spark nodes with 128 GB of memory per node and the ConnectX/RoCE network configured as described below.", "Docker with NVIDIA GPU support and the image and checkpoint cached on both nodes." ], "limitations": [ "The source requires the custom B12x container image." ], "checkpoint": { "target": "https://huggingface.co/nvidia/GLM-5.3-Flash-FP4-Local", "target_label": "nvidia/GLM-5.3-Flash-FP4-Local", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× DGX Spark; one node rank per Spark; tensor parallel 2", "gpu_count": 2, "node_count": 2, "tensor_parallel_size": 2 }, "runtime": { "kind": "container", "label": "Container (community, ARM64, custom B12x)", "image": "eugr/spark-vllm-b12x:nightly-20261015", "image_url": "https://hub.docker.com/layers/eugr/spark-vllm-b12x/nightly-20261015", "setup_label": "Start container", "setup_target": "all-nodes", "setup_instructions": "Run on every node after setting `HF_TOKEN`; replace the node-specific placeholders.", "setup_command": "# Example two-Spark values (adjust to your network):\n# HEAD_IP=\"10.100.32.1\" # head node IP on the ETH subnet\n# ETH_IF=\"enp1s0f0np0\" # ConnectX baseboard ethernet (socket/control)\n# IB_IF=\"rocep1s0f0,roceP2p1s0f0\" # RoCE HCA names (data plane) - use both \"twins\"!\nexport HEAD_IP=\"\"\nexport ETH_IF=\"\"\nexport IB_IF=\",\"\nexport MASTER_PORT=29501\nexport IMAGE=\"eugr/spark-vllm-b12x:nightly-20261015\"\nexport HF_CACHE_DIR=\"$HOME/.cache/huggingface\"\n\nmkdir -p \"$HF_CACHE_DIR\" \"$HOME/.cache/vllm\" \"$HOME/.cache/flashinfer\" \"$HOME/.triton\" \"$HOME/.cache/b12x\"\n\ndocker run --rm -it \\\n --privileged \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit nofile=1048576:1048576 \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HF_CACHE_DIR:/root/.cache/huggingface\" \\\n -v \"$HOME/.cache/vllm:/root/.cache/vllm\" \\\n -v \"$HOME/.cache/flashinfer:/root/.cache/flashinfer\" \\\n -v \"$HOME/.triton:/root/.triton\" \\\n -v $HOME/.cache/b12x:/root/.cache/b12x \\\n --entrypoint /bin/bash \\\n \"$IMAGE\"" }, "serve": { "label": "Launch head", "target": "head", "instructions": null, "command": "export CUTE_DSL_ARCH=\"sm_121a\"\nexport SAFETENSORS_FAST_GPU=\"1\"\nexport VLLM_ENABLE_ROCE_ALLREDUCE=\"1\"\nexport VLLM_ROCE_ALLREDUCE_MAX_SIZE=\"2MB\"\nexport NCCL_BUFFSIZE=1048576\nexport NCCL_MAX_CTAS=8\nexport VLLM_WORKER_MULTIPROC_METHOD=\"spawn\"\nexport VLLM_SSM_CONV_STATE_LAYOUT=\"DS\"\nexport VLLM_USE_AOT_COMPILE=\"1\"\nexport VLLM_USE_MEGA_AOT_ARTIFACT=\"1\"\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\nexport VLLM_ENABLE_PCIE_ALLREDUCE=\"0\"\nexport B12X_POLICY_MODE=\"auto\"\nexport VLLM_B12X_MOE_FP4_FORCE_A16=1\nexport B12X_W4A16_A4_PREFILL=1\n\nvllm serve nvidia/GLM-5.3-Flash-FP4-Local \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --tensor-parallel-size 2 \\\n --pipeline-parallel-size 1 \\\n --decode-context-parallel-size 1 \\\n --mamba-cache-mode align \\\n --enable-prefix-caching \\\n --enable-chunked-prefill \\\n --dtype bfloat16 \\\n --kv-cache-dtype fp8 \\\n --quantization modelopt_mixed \\\n --attention-backend B12X \\\n --block-size 256 \\\n --moe-backend b12x \\\n --linear-backend b12x \\\n --no-enable-flashinfer-autotune \\\n --load-format b12x \\\n --max-model-len 500000 \\\n --max-num-seqs 4 \\\n --max-num-batched-tokens 4096 \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":5,\"moe_backend\":\"humming\",\"attention_backend\":\"B12X\"}' \\\n --reasoning-parser glm45 \\\n --tool-call-parser glm47 \\\n --enable-auto-tool-choice \\\n --gpu-memory-utilization 0.87 \\\n --nnodes 2 \\\n --node-rank 0 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\"", "port": 8000, "api_model": "nvidia/GLM-5.3-Flash-FP4-Local", "additional_commands": [ { "id": "worker", "label": "Launch worker", "target": "workers", "instructions": null, "command": "export CUTE_DSL_ARCH=\"sm_121a\"\nexport SAFETENSORS_FAST_GPU=\"1\"\nexport VLLM_ENABLE_ROCE_ALLREDUCE=\"1\"\nexport VLLM_ROCE_ALLREDUCE_MAX_SIZE=\"2MB\"\nexport NCCL_BUFFSIZE=1048576\nexport NCCL_MAX_CTAS=8\nexport VLLM_WORKER_MULTIPROC_METHOD=\"spawn\"\nexport VLLM_SSM_CONV_STATE_LAYOUT=\"DS\"\nexport VLLM_USE_AOT_COMPILE=\"1\"\nexport VLLM_USE_MEGA_AOT_ARTIFACT=\"1\"\nexport VLLM_USE_V2_MODEL_RUNNER=\"1\"\nexport VLLM_ENABLE_PCIE_ALLREDUCE=\"0\"\nexport B12X_POLICY_MODE=\"auto\"\nexport VLLM_B12X_MOE_FP4_FORCE_A16=1\nexport B12X_W4A16_A4_PREFILL=1\n\nvllm serve nvidia/GLM-5.3-Flash-FP4-Local \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --tensor-parallel-size 2 \\\n --pipeline-parallel-size 1 \\\n --decode-context-parallel-size 1 \\\n --mamba-cache-mode align \\\n --enable-prefix-caching \\\n --enable-chunked-prefill \\\n --dtype bfloat16 \\\n --kv-cache-dtype fp8 \\\n --quantization modelopt_mixed \\\n --attention-backend B12X \\\n --block-size 256 \\\n --moe-backend b12x \\\n --linear-backend b12x \\\n --no-enable-flashinfer-autotune \\\n --load-format b12x \\\n --max-model-len 500000 \\\n --max-num-seqs 4 \\\n --max-num-batched-tokens 4096 \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":5,\"moe_backend\":\"humming\",\"attention_backend\":\"B12X\"}' \\\n --reasoning-parser glm45 \\\n --tool-call-parser glm47 \\\n --enable-auto-tool-choice \\\n --gpu-memory-utilization 0.87 \\\n --nnodes 2 \\\n --node-rank 1 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\" \\\n --headless" } ] }, "launch": null, "sampling": null, "notes": "Use the same CX7 port on every node; assign both port subinterfaces on separate subnets; cache the image and checkpoint on every node; start workers before the head. Docker runs privileged.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/GLM-5.3-Flash-FP4-Local\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/GLM-5.3-Flash-FP4-Local\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "z-ai/glm-5.3-flash/dgx-station/linux/vllm/nvfp4/mtp", "hardware": "dgx-station", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "v0.31.0", "backend_min_version": null, "min_vidmem_gb": null, "precision": "nvfp4", "variant": "mtp", "status": "experimental", "prerequisites": [ "One DGX Station with a GB300 and about 200 GB of free disk for the checkpoint.", "Docker with the NVIDIA container runtime and permission to select the GB300 by UUID." ], "limitations": [ "Validated with the RedHatAI checkpoint only. On v0.30.0, nvidia/GLM-5.3-Flash-NVFP4 failed to load its MTP draft layer (MoE weight shape mismatch) and served only without `--speculative-config`; not re-tested on v0.31.0." ], "checkpoint": { "target": "https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4", "target_label": "RedHatAI/GLM-5.3-Flash-NVFP4", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "1× DGX Station; tensor parallel 1", "gpu_count": 1, "node_count": 1, "tensor_parallel_size": 1 }, "runtime": { "kind": "container", "label": "Container (ARM64)", "image": "vllm/vllm-openai:v0.31.0-aarch64", "image_url": "https://hub.docker.com/r/vllm/vllm-openai/tags?name=v0.31.0-aarch64", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment (optional; the checkpoint is public). The DGX Station also carries an auxiliary display GPU, so the container is pinned to the GB300 by UUID.", "setup_command": "GPU_UUID=$(nvidia-smi --query-gpu=uuid,name --format=csv,noheader | \\\n awk -F, '$2 ~ /GB300/ {gsub(/ /,\"\",$1); print $1; exit}')\ntest -n \"$GPU_UUID\"\n\ndocker run --privileged --rm -it \\\n --gpus device=\"$GPU_UUID\" \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai:v0.31.0-aarch64" }, "serve": { "label": "Launch", "target": "single-node", "instructions": "First launch downloads 198 GB to ~/.cache/huggingface.", "command": "hf download RedHatAI/GLM-5.3-Flash-NVFP4 --revision 18d55bfd5a2194887738da73753975c9d3842f46 --local-dir /root/.cache/huggingface/GLM-5.3-Flash-NVFP4\nexport HF_HUB_OFFLINE=1\n\nvllm serve /root/.cache/huggingface/GLM-5.3-Flash-NVFP4 \\\n --served-model-name RedHatAI/GLM-5.3-Flash-NVFP4 \\\n --tensor-parallel-size 1 \\\n --max-num-seqs 32 \\\n --max-num-batched-tokens 8192 \\\n --gpu-memory-utilization 0.95 \\\n --tool-call-parser glm47 \\\n --enable-auto-tool-choice \\\n --reasoning-parser glm45 \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":5}'", "port": 8000, "api_model": "RedHatAI/GLM-5.3-Flash-NVFP4" }, "launch": null, "sampling": null, "notes": "Serves the full 1,048,576-token context with BF16 KV cache. At `--gpu-memory-utilization 0.95` the KV pool is about 2.9M tokens and all 32 sequences run concurrently with MTP; at the 0.92 default it is about 2.3M tokens and at most 28 run, which queues requests at 32 users (TTFT 2.7 s versus 0.46 s on the synthetic 2K/1K shape). Thinking is on by default; send `\"chat_template_kwargs\": {\"enable_thinking\": false}` to disable it. Startup takes about 8 minutes with the weights cached.", "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"RedHatAI/GLM-5.3-Flash-NVFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"RedHatAI/GLM-5.3-Flash-NVFP4\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "z-ai/glm-5.3-flash/rtx-pro-2gpu/linux/vllm/fp4-local", "hardware": "rtx-pro-2gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21830+g8244d1c44", "backend_min_version": null, "min_vidmem_gb": 96, "precision": "fp4-local", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/GLM-5.3-Flash-FP4-Local", "target_label": "nvidia/GLM-5.3-Flash-FP4-Local", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "2× RTX Pro", "gpu_count": 2, "node_count": 1, "tensor_parallel_size": 2 }, "runtime": { "kind": "container", "label": "Container (community)", "image": "ghcr.io/local-inference-lab/vllm:release-10152026", "image_url": "https://github.com/local-inference-lab/blackwell-llm-docker/pkgs/container/vllm/1392689728?tag=release-10152026", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n ghcr.io/local-inference-lab/vllm:release-10152026" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "export B12X_DYNAMIC_DIRECT_EXPERT_SCALES=1\nexport B12X_DYNAMIC_SKIP_SPLIT_BARRIER_RESET=1\nexport B12X_DYNAMIC_SPLIT_COMPUTE_MAC=224\nexport B12X_DYNAMIC_SPLIT_FAST_PREPARE=1\nexport B12X_DYNAMIC_SPLIT_LOW_SMEM=1\nexport B12X_DYNAMIC_SPLIT_ROUTE_COMPUTE=1\nexport B12X_DYNAMIC_WORK_SOURCE=persistent_grid\nexport B12X_MHC_PDL=1\nexport B12X_PCIE_ONESHOT_BLOCK_LIMIT=4\nexport B12X_PCIE_ONESHOT_PDL=1\nexport B12X_PCIE_ONESHOT_THREADS=512\nexport B12X_W4A16_A4_PREFILL=1\nexport B12X_W4A16_FP32_TOPK_WEIGHTS=1\nexport CUBLAS_WORKSPACE_CONFIG=:4096:1\nexport CUDA_DEVICE_ORDER=PCI_BUS_ID\nexport CUTE_DSL_ARCH=sm_120a\nexport GLOO_SOCKET_IFNAME=lo\nexport INSTANTTENSOR_BACKEND=BUFFERED\nexport NCCL_BUFFSIZE=1048576\nexport NCCL_IB_DISABLE=1\nexport NCCL_MAX_NCHANNELS=2\nexport NCCL_MIN_NCHANNELS=2\nexport NCCL_NET_PLUGIN=none\nexport NCCL_P2P_LEVEL=SYS\nexport NCCL_PROTO=LL,LL128,Simple\nexport NCCL_SOCKET_IFNAME=lo\nexport NCCL_TUNER_PLUGIN=none\nexport OMP_NUM_THREADS=1\nexport PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,large_segment_size_mb:12\nexport SAFETENSORS_FAST_GPU=1\nexport TORCH_CUDA_ARCH_LIST=12.0a\nexport VLLM_B12X_MLA_CKV_GATHER=1\nexport VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS=65536\nexport VLLM_B12X_MOE_FP4_FORCE_A16=1\nexport VLLM_CAUSAL_CONV1D_UPDATE_HOIST=1\nexport VLLM_DISABLED_KERNELS=MarlinFP8ScaledMMLinearKernel\nexport VLLM_DISABLE_SHARED_EXPERTS_STREAM=0\nexport VLLM_ENABLE_PCIE_ALLREDUCE=1\nexport VLLM_GLM53_DFLASH_ATTN=1\nexport VLLM_GLM53_EMBED_HOST=1\nexport VLLM_GLM53_KDA_GATE_SIDE_STREAM=1\nexport VLLM_GLM53_KDA_MXFP8_ACTIVATION_MODE=prefill\nexport VLLM_GLM53_L2_PREFETCH=1\nexport VLLM_GLM53_L2_PREFETCH_PERSIST_MB=0\nexport VLLM_GLM53_MTP_DRAFT_HEAD=bf16\nexport VLLM_GLM53_ONLINE_DENSE_MXFP8=0\nexport VLLM_GLM53_SPLIT_MAMBA_BLOCK_SIZE=auto\nexport VLLM_GLM53_SPLIT_TARGET_BLOCK_SIZE=2048\nexport VLLM_LM_HEAD_A16=1\nexport VLLM_MTP_NVFP4_LM_HEAD=0\nexport VLLM_MXFP8_LM_HEAD=0\nexport VLLM_PCIE_ALLREDUCE_BACKEND=b12x\nexport VLLM_PCIE_DMA_MIN_BYTES=6MB\nexport VLLM_PCIE_TWOSHOT_ALLREDUCE_MAX_SIZE=off\nexport VLLM_PLUGINS=''\nexport VLLM_SHARE_PYNCCL_COMMS=1\nexport VLLM_USE_FASTOKENS=1\nexport VLLM_USE_FLASHINFER_SAMPLER=1\nexport VLLM_USE_V2_MODEL_RUNNER=1\nexport VLLM_WORKER_MULTIPROC_METHOD=spawn\nvllm serve nvidia/GLM-5.3-Flash-FP4-Local \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --additional-config '{\"glm53_kda_decode_backend\":\"auto\",\"kda_prefill_backend\":\"b12x\"}' \\\n --attention-backend B12X \\\n --block-size 256 \\\n --chat-template /opt/lil/runtime/templates/glm53-flash.jinja \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --cp-kv-cache-interleave-size 4 \\\n --cudagraph-capture-sizes 1 2 4 8 12 16 20 24 28 32 \\\n --dcp-kv-cache-interleave-size 4 \\\n --decode-context-parallel-size 2 \\\n --default-chat-template-kwargs '{\"reasoning_effort\":\"max\",\"clear_thinking\":false}' \\\n --dtype bfloat16 \\\n --enable-auto-tool-choice \\\n --enable-chunked-prefill \\\n --no-enable-flashinfer-autotune \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --gpu-memory-utilization 0.985 \\\n --kv-cache-dtype fp8 \\\n --kv-cache-memory-bytes 7650410496 \\\n --linear-backend b12x \\\n --load-format instanttensor \\\n --mamba-cache-mode align \\\n --max-cudagraph-capture-size 32 \\\n --max-model-len 1048576 \\\n --max-num-batched-tokens 4096 \\\n --max-num-seqs 8 \\\n --max-parallel-prefills 1 \\\n --moe-backend b12x \\\n --override-generation-config '{\"temperature\":1.0,\"top_p\":0.95}' \\\n --prefill-compute-share 0.4 \\\n --prefill-schedule-interval 1 \\\n --quantization modelopt_mixed \\\n --reasoning-parser glm45 \\\n --recurrent-checkpoint-policy request_boundaries \\\n --speculative-config '{\"method\":\"mtp\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"standard\",\"moe_backend\":\"b12x\",\"attention_backend\":\"B12X\",\"num_speculative_tokens\":3}' \\\n --tensor-parallel-size 2 \\\n --tool-call-parser glm47", "port": 8000, "api_model": "nvidia/GLM-5.3-Flash-FP4-Local" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/GLM-5.3-Flash-FP4-Local\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/GLM-5.3-Flash-FP4-Local\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } }, { "id": "z-ai/glm-5.3-flash/rtx-pro-4gpu/linux/vllm/fp4-local", "hardware": "rtx-pro-4gpu", "platform": { "os": "linux", "execution": "container" }, "backend": "vllm", "backend_version": "0.1.dev21830+g8244d1c44.cu134", "backend_min_version": null, "min_vidmem_gb": null, "precision": "fp4-local", "variant": null, "status": "optimized", "prerequisites": [], "limitations": [], "checkpoint": { "target": "https://huggingface.co/nvidia/GLM-5.3-Flash-FP4-Local", "target_label": "nvidia/GLM-5.3-Flash-FP4-Local", "draft": null, "draft_label": null }, "security": "Evaluation configuration only. Host networking exposes vLLM on host interfaces; use a firewall or reverse proxy, a scoped read-only HF token, and an isolated cache. For production, pin tested image, model, and code revisions.", "topology": { "summary": "4× RTX Pro 6000", "gpu_count": 4, "node_count": 1, "tensor_parallel_size": 4 }, "runtime": { "kind": "container", "label": "Container (community)", "image": "ghcr.io/local-inference-lab/vllm:release-10152026", "image_url": "https://github.com/local-inference-lab/blackwell-llm-docker/pkgs/container/vllm/1392689728?tag=release-10152026", "setup_label": "Start container", "setup_target": "single-node", "setup_instructions": "Set `HF_TOKEN` in the host environment.", "setup_command": "docker run --rm -it \\\n --gpus all \\\n --ipc host \\\n --network host \\\n --shm-size=16g \\\n --ulimit memlock=-1 \\\n --ulimit stack=67108864 \\\n -e HF_TOKEN=$HF_TOKEN \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n --entrypoint /bin/bash \\\n ghcr.io/local-inference-lab/vllm:release-10152026" }, "serve": { "label": "Launch", "target": "single-node", "instructions": null, "command": "export B12X_DYNAMIC_DIRECT_EXPERT_SCALES=1\nexport B12X_DYNAMIC_SKIP_SPLIT_BARRIER_RESET=1\nexport B12X_DYNAMIC_SPLIT_COMPUTE_MAC=224\nexport B12X_DYNAMIC_SPLIT_FAST_PREPARE=1\nexport B12X_DYNAMIC_SPLIT_LOW_SMEM=1\nexport B12X_DYNAMIC_SPLIT_ROUTE_COMPUTE=1\nexport B12X_DYNAMIC_WORK_SOURCE=persistent_grid\nexport B12X_MHC_PDL=1\nexport B12X_PCIE_ONESHOT_BLOCK_LIMIT=4\nexport B12X_PCIE_ONESHOT_PDL=1\nexport B12X_PCIE_ONESHOT_THREADS=512\nexport B12X_W4A16_A4_PREFILL=1\nexport B12X_W4A16_FP32_TOPK_WEIGHTS=1\nexport CUBLAS_WORKSPACE_CONFIG=:4096:1\nexport CUDA_DEVICE_ORDER=PCI_BUS_ID\nexport CUTE_DSL_ARCH=sm_120a\nexport GLOO_SOCKET_IFNAME=lo\nexport INSTANTTENSOR_BACKEND=BUFFERED\nexport NCCL_BUFFSIZE=1048576\nexport NCCL_IB_DISABLE=1\nexport NCCL_MAX_NCHANNELS=2\nexport NCCL_MIN_NCHANNELS=2\nexport NCCL_NET_PLUGIN=none\nexport NCCL_P2P_LEVEL=SYS\nexport NCCL_PROTO=LL,LL128,Simple\nexport NCCL_SOCKET_IFNAME=lo\nexport NCCL_TUNER_PLUGIN=none\nexport OMP_NUM_THREADS=1\nexport PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,large_segment_size_mb:12\nexport SAFETENSORS_FAST_GPU=1\nexport TORCH_CUDA_ARCH_LIST=12.0a\nexport VLLM_B12X_MLA_CKV_GATHER=1\nexport VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS=65536\nexport VLLM_B12X_MOE_FP4_FORCE_A16=1\nexport VLLM_CAUSAL_CONV1D_UPDATE_HOIST=1\nexport VLLM_DISABLED_KERNELS=MarlinFP8ScaledMMLinearKernel\nexport VLLM_DISABLE_SHARED_EXPERTS_STREAM=0\nexport VLLM_ENABLE_PCIE_ALLREDUCE=1\nexport VLLM_GLM53_DFLASH_ATTN=1\nexport VLLM_GLM53_EMBED_HOST=1\nexport VLLM_GLM53_KDA_GATE_SIDE_STREAM=1\nexport VLLM_GLM53_KDA_MXFP8_ACTIVATION_MODE=prefill\nexport VLLM_GLM53_L2_PREFETCH=1\nexport VLLM_GLM53_L2_PREFETCH_PERSIST_MB=0\nexport VLLM_GLM53_MTP_DRAFT_HEAD=bf16\nexport VLLM_GLM53_ONLINE_DENSE_MXFP8=0\nexport VLLM_GLM53_SPLIT_MAMBA_BLOCK_SIZE=auto\nexport VLLM_GLM53_SPLIT_TARGET_BLOCK_SIZE=2048\nexport VLLM_LM_HEAD_A16=1\nexport VLLM_MTP_NVFP4_LM_HEAD=0\nexport VLLM_MXFP8_LM_HEAD=0\nexport VLLM_PCIE_ALLREDUCE_BACKEND=b12x\nexport VLLM_PCIE_DMA_MIN_BYTES=6MB\nexport VLLM_PCIE_TWOSHOT_ALLREDUCE_MAX_SIZE=off\nexport VLLM_PLUGINS=''\nexport VLLM_SHARE_PYNCCL_COMMS=1\nexport VLLM_USE_FASTOKENS=1\nexport VLLM_USE_FLASHINFER_SAMPLER=1\nexport VLLM_USE_V2_MODEL_RUNNER=1\nexport VLLM_WORKER_MULTIPROC_METHOD=spawn\n\nvllm serve nvidia/GLM-5.3-Flash-FP4-Local \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --additional-config '{\"glm53_kda_decode_backend\":\"auto\",\"kda_prefill_backend\":\"b12x\"}' \\\n --attention-backend B12X \\\n --block-size 256 \\\n --chat-template /opt/lil/runtime/templates/glm53-flash.jinja \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --cudagraph-capture-sizes 1 2 4 8 12 16 20 24 28 32 \\\n --default-chat-template-kwargs '{\"reasoning_effort\":\"max\",\"clear_thinking\":false}' \\\n --dtype bfloat16 \\\n --enable-auto-tool-choice \\\n --enable-chunked-prefill \\\n --no-enable-flashinfer-autotune \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --gpu-memory-utilization 0.95 \\\n --kv-cache-dtype fp8 \\\n --linear-backend b12x \\\n --load-format instanttensor \\\n --mamba-cache-mode align \\\n --max-cudagraph-capture-size 32 \\\n --max-model-len 1048576 \\\n --max-num-batched-tokens 4096 \\\n --max-num-seqs 16 \\\n --moe-backend b12x \\\n --override-generation-config '{\"temperature\":1.0,\"top_p\":0.95}' \\\n --prefill-compute-share 0.4 \\\n --prefill-schedule-interval 1 \\\n --quantization modelopt_mixed \\\n --reasoning-parser glm45 \\\n --recurrent-checkpoint-policy request_boundaries \\\n --speculative-config '{\"method\":\"mtp\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"standard\",\"moe_backend\":\"b12x\",\"attention_backend\":\"B12X\",\"num_speculative_tokens\":3}' \\\n --tensor-parallel-size 4 \\\n --tool-call-parser glm47", "port": 8000, "api_model": "nvidia/GLM-5.3-Flash-FP4-Local" }, "launch": null, "sampling": null, "notes": null, "clients": { "shell": "curl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\"model\": \"nvidia/GLM-5.3-Flash-FP4-Local\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello\"}]}'", "python": "from openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresp = client.chat.completions.create(\n model=\"nvidia/GLM-5.3-Flash-FP4-Local\",\n messages=[{\"role\": \"user\", \"content\": \"Hello\"}],\n)\nprint(resp.choices[0].message.content)" } } ] } ] }