| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | import os |
| 6 | from typing import Any |
| 7 | |
| 8 | import pytest |
| 9 | |
| 10 | from tests.utils import RemoteOpenAIServer |
| 11 | from vllm.platforms import current_platform |
| 12 | |
| 13 | if not current_platform.is_device_capability_family(100): |
| 14 | pytest.skip( |
| 15 | "This test only runs on Blackwell GPUs (SM10x).", allow_module_level=True |
| 16 | ) |
| 17 | |
| 18 | |
| 19 | @pytest.fixture(scope="module", autouse=True) |
| 20 | def set_test_environment(): |
| 21 | """Sets environment variables required for this test module.""" |
| 22 | # Make sure TRTLLM attention is available |
| 23 | os.environ["VLLM_HAS_FLASHINFER_CUBIN"] = "1" |
| 24 | # Set compilation threads to 16 to speed up startup |
| 25 | os.environ["FLASHINFER_NVCC_THREADS"] = "16" |
| 26 | |
| 27 | |
| 28 | # Override the backbone layers to 4 for faster startup |
| 29 | HF_OVERRIDE_TEXT = { |
| 30 | "num_layers": 4, |
| 31 | "num_hidden_layers": 4, |
| 32 | } |
| 33 | HF_OVERRIDE_MM = { |
| 34 | "text_config": {"num_layers": 4, "num_hidden_layers": 4}, |
| 35 | } |
| 36 | |
| 37 | |
| 38 | def can_initialize( |
| 39 | model: str, |
| 40 | hf_overrides: dict[str, Any] | None = None, |
| 41 | extra_args: list[str] | None = None, |
| 42 | ): |
| 43 | # Server arguments |
| 44 | extra_args = extra_args if extra_args is not None else [] |
| 45 | server_args = [ |
| 46 | "--max-model-len", |
| 47 | "2048", |
| 48 | "--max-num-batched-tokens", |
| 49 | "256", |
| 50 | "--load-format", |
| 51 | "dummy", |
| 52 | "--trust-remote-code", |
| 53 | "--limit-mm-per-prompt", |
| 54 | json.dumps({"image": 0}), |
| 55 | *extra_args, |
| 56 | ] |
| 57 | |
| 58 | # Launch server and make a simple request |
| 59 | with RemoteOpenAIServer( |
| 60 | model, |
| 61 | server_args, |
| 62 | max_wait_seconds=1500, # Due to FlashInfer compile |
| 63 | override_hf_configs=hf_overrides, |
| 64 | ) as server: |
| 65 | client = server.get_client() |
| 66 | # Make a simple request to verify the server works |
| 67 | completion = client.completions.create( |
| 68 | model=model, |
| 69 | prompt=["Hello, World!"], |
| 70 | temperature=0, |
| 71 | max_tokens=2, |
| 72 | ) |
| 73 | print(completion) |
| 74 | assert completion.choices[0].text is not None |
| 75 | |
| 76 | |
| 77 | ## Llama4 ## |
| 78 | |
| 79 | |
| 80 | @pytest.mark.skip( |
| 81 | reason=( |
| 82 | "RuntimeError: run_moe() Expected a value of type " |
| 83 | "'Optional[List[Tensor]]' for argument '_9' but instead found type " |
| 84 | "'list'." |
| 85 | ) |
| 86 | ) |
| 87 | def test_llama4_fp8_tensor_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch): |
| 88 | can_initialize( |
| 89 | "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8", |
| 90 | hf_overrides=HF_OVERRIDE_MM, |
| 91 | extra_args=["--moe-backend=flashinfer_cutlass"], |
| 92 | ) |
| 93 | |
| 94 | |
| 95 | def test_llama4_fp8_tensor_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch): |
| 96 | can_initialize( |
| 97 | "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8", |
| 98 | hf_overrides=HF_OVERRIDE_MM, |
| 99 | extra_args=["--moe-backend=flashinfer_trtllm"], |
| 100 | ) |
| 101 | |
| 102 | |
| 103 | def test_llama4_nvfp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch): |
| 104 | can_initialize( |
| 105 | "nvidia/Llama-4-Scout-17B-16E-Instruct-FP4", |
| 106 | hf_overrides=HF_OVERRIDE_MM, |
| 107 | extra_args=["--moe-backend=flashinfer_cutlass"], |
| 108 | ) |
| 109 | |
| 110 | |
| 111 | def test_llama4_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch): |
| 112 | can_initialize( |
| 113 | "nvidia/Llama-4-Scout-17B-16E-Instruct-FP4", |
| 114 | hf_overrides=HF_OVERRIDE_MM, |
| 115 | extra_args=["--moe-backend=flashinfer_trtllm"], |
| 116 | ) |
| 117 | |
| 118 | |
| 119 | ## DeepSeekV3 ## |
| 120 | |
| 121 | |
| 122 | def test_deepseek_fp8_block_moe_deep_gemm(monkeypatch: pytest.MonkeyPatch): |
| 123 | can_initialize( |
| 124 | "deepseek-ai/DeepSeek-V3.1", |
| 125 | hf_overrides=HF_OVERRIDE_TEXT, |
| 126 | extra_args=["--moe-backend=deep_gemm"], |
| 127 | ) |
| 128 | |
| 129 | |
| 130 | def test_deepseek_fp8_block_moe_vllm_triton(monkeypatch: pytest.MonkeyPatch): |
| 131 | can_initialize( |
| 132 | "deepseek-ai/DeepSeek-V3.1", |
| 133 | hf_overrides=HF_OVERRIDE_TEXT, |
| 134 | extra_args=["--moe-backend=triton"], |
| 135 | ) |
| 136 | |
| 137 | |
| 138 | @pytest.mark.skip( |
| 139 | reason=( |
| 140 | "Known issue: lack of kernel support. " |
| 141 | "Expected failure: assert self.block_quant is None" |
| 142 | ) |
| 143 | ) |
| 144 | def test_deepseek_fp8_block_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch): |
| 145 | can_initialize( |
| 146 | "deepseek-ai/DeepSeek-V3.1", |
| 147 | hf_overrides=HF_OVERRIDE_TEXT, |
| 148 | extra_args=["--moe-backend=flashinfer_cutlass"], |
| 149 | ) |
| 150 | |
| 151 | |
| 152 | def test_deepseek_fp8_block_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch): |
| 153 | can_initialize( |
| 154 | "deepseek-ai/DeepSeek-V3.1", |
| 155 | hf_overrides=HF_OVERRIDE_TEXT, |
| 156 | extra_args=["--moe-backend=flashinfer_trtllm"], |
| 157 | ) |
| 158 | |
| 159 | |
| 160 | def test_deepseek_nvfp4_moe_flashinfer_vllm(monkeypatch: pytest.MonkeyPatch): |
| 161 | can_initialize( |
| 162 | "nvidia/DeepSeek-R1-0528-FP4-v2", |
| 163 | hf_overrides=HF_OVERRIDE_TEXT, |
| 164 | extra_args=["--moe-backend=cutlass"], |
| 165 | ) |
| 166 | |
| 167 | |
| 168 | def test_deepseek_nvfp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch): |
| 169 | can_initialize( |
| 170 | "nvidia/DeepSeek-R1-0528-FP4-v2", |
| 171 | hf_overrides=HF_OVERRIDE_TEXT, |
| 172 | extra_args=["--moe-backend=flashinfer_cutlass"], |
| 173 | ) |
| 174 | |
| 175 | |
| 176 | def test_deepseek_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch): |
| 177 | can_initialize( |
| 178 | "nvidia/DeepSeek-R1-0528-FP4-v2", |
| 179 | hf_overrides=HF_OVERRIDE_TEXT, |
| 180 | extra_args=["--moe-backend=flashinfer_trtllm"], |
| 181 | ) |
| 182 | |
| 183 | |
| 184 | ## GPT-OSS ## |
| 185 | |
| 186 | |
| 187 | def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch): |
| 188 | can_initialize( |
| 189 | "openai/gpt-oss-20b", |
| 190 | hf_overrides=HF_OVERRIDE_TEXT, |
| 191 | extra_args=["--moe-backend=flashinfer_trtllm"], |
| 192 | ) |
| 193 | |
| 194 | |
| 195 | def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch): |
| 196 | can_initialize( |
| 197 | "openai/gpt-oss-20b", |
| 198 | hf_overrides=HF_OVERRIDE_TEXT, |
| 199 | extra_args=[ |
| 200 | "--moe-backend", |
| 201 | "flashinfer_cutlass", |
| 202 | "--quantization-config.moe.activation", |
| 203 | "mxfp8", |
| 204 | ], |
| 205 | ) |
| 206 | |
| 207 | |
| 208 | def test_gptoss_mxfp4mxfp8_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch): |
| 209 | can_initialize( |
| 210 | "openai/gpt-oss-20b", |
| 211 | hf_overrides=HF_OVERRIDE_TEXT, |
| 212 | extra_args=[ |
| 213 | "--quantization-config.moe.activation", |
| 214 | "mxfp8", |
| 215 | ], |
| 216 | ) |
| 217 | |
| 218 | |
| 219 | def test_gptoss_eager(monkeypatch: pytest.MonkeyPatch): |
| 220 | can_initialize( |
| 221 | "openai/gpt-oss-20b", |
| 222 | hf_overrides=HF_OVERRIDE_TEXT, |
| 223 | extra_args=["--enforce-eager"], |
| 224 | ) |
| 225 | |
| 226 | |
| 227 | ## Qwen3 Next ## |
| 228 | |
| 229 | |
| 230 | @pytest.mark.skip( |
| 231 | reason=( |
| 232 | "FLASHINFER TRTLLM MoE has a bug with all negative router logits " |
| 233 | "for models with RENORMALIZE. This will be re-enabled once the " |
| 234 | "issue is fixed in flashinfer." |
| 235 | ) |
| 236 | ) |
| 237 | def test_qwen3_next_bf16_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch): |
| 238 | can_initialize( |
| 239 | "Qwen/Qwen3-Next-80B-A3B-Instruct", |
| 240 | hf_overrides=HF_OVERRIDE_TEXT, |
| 241 | extra_args=["--moe-backend=flashinfer_trtllm"], |
| 242 | ) |
| 243 | |
| 244 | |
| 245 | ## NemoTron ## |
| 246 | |
| 247 | |
| 248 | def test_nemotron_fp8_moe_flashinfer_throughput(monkeypatch: pytest.MonkeyPatch): |
| 249 | can_initialize( |
| 250 | "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", |
| 251 | hf_overrides=HF_OVERRIDE_TEXT, |
| 252 | extra_args=["--moe-backend=flashinfer_cutlass"], |
| 253 | ) |
| 254 | |
| 255 | |
| 256 | @pytest.mark.skip( |
| 257 | reason=( |
| 258 | "FP8 MoE backend FLASHINFER_TRTLLM does not support the " |
| 259 | "deployment configuration since kernel does not support " |
| 260 | "no act_and_mul MLP layer." |
| 261 | ) |
| 262 | ) |
| 263 | def test_nemotron_fp8_moe_flashinfer_latency(monkeypatch: pytest.MonkeyPatch): |
| 264 | can_initialize( |
| 265 | "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", |
| 266 | hf_overrides=HF_OVERRIDE_TEXT, |
| 267 | extra_args=["--moe-backend=flashinfer_trtllm"], |
| 268 | ) |
| 269 | |
| 270 | |
| 271 | @pytest.mark.skip( |
| 272 | reason=( |
| 273 | "FP8 MoE backend TRITON does not support the " |
| 274 | "deployment configuration since kernel does not support " |
| 275 | "no act_and_mul MLP layer." |
| 276 | ) |
| 277 | ) |
| 278 | def test_nemotron_fp8_moe_vllm_triton(monkeypatch: pytest.MonkeyPatch): |
| 279 | can_initialize( |
| 280 | "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", |
| 281 | hf_overrides=HF_OVERRIDE_TEXT, |
| 282 | extra_args=["--moe-backend=triton"], |
| 283 | ) |
| 284 | |
| 285 | |
| 286 | def test_nemotron_fp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch): |
| 287 | can_initialize( |
| 288 | "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4", |
| 289 | hf_overrides=HF_OVERRIDE_TEXT, |
| 290 | extra_args=["--moe-backend=flashinfer_cutlass"], |
| 291 | ) |
| 292 | |
| 293 | |
| 294 | def test_nemotron_fp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch): |
| 295 | can_initialize( |
| 296 | "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4", |
| 297 | hf_overrides=HF_OVERRIDE_TEXT, |
| 298 | extra_args=["--moe-backend=flashinfer_trtllm"], |
| 299 | ) |