A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI] Add Blackwell DeepSeek FP8 FlashInfer MoE tests (#26040)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Oct 2, 2025 at 12:07 UTC
3b279a84be27742fc3819a9942a77a29a399f9da
1 file changed
+29
-11
tests/quantization/test_blackwell_moe.py
+29
-11
@@ -3,6 +3,7 @@
3
4
import json
5
import os
6
+from typing import Optional
7
8
import pytest
9
@@ -20,9 +21,10 @@ os.environ["FLASHINFER_NVCC_THREADS"] = "16"
21
dummy_hf_overrides = {"num_layers": 4, "num_hidden_layers": 4}
22
23
23
-def can_initialize(model: str, extra_args: list[str]):
24
+def can_initialize(model: str, extra_args: Optional[list[str]] = None):
25
26
# Server arguments
27
+ extra_args = extra_args if extra_args is not None else []
28
server_args = [
29
"--max-model-len",
30
"2048",
@@ -65,7 +67,7 @@ def test_llama4_fp8_tensor_moe_flashinfer_cutlass(
67
monkeypatch: pytest.MonkeyPatch):
68
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP8", "1")
69
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")
68
- can_initialize("nvidia/Llama-4-Scout-17B-16E-Instruct-FP8", [])
70
+ can_initialize("nvidia/Llama-4-Scout-17B-16E-Instruct-FP8")
71
72
73
@pytest.mark.skip(reason="Works, but takes too long to run")
@@ -73,21 +75,21 @@ def test_llama4_fp8_tensor_moe_flashinfer_trtllm(
75
monkeypatch: pytest.MonkeyPatch):
76
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP8", "1")
77
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "latency")
76
- can_initialize("nvidia/Llama-4-Scout-17B-16E-Instruct-FP8", [])
78
+ can_initialize("nvidia/Llama-4-Scout-17B-16E-Instruct-FP8")
79
80
81
@pytest.mark.skip(reason="Works, but takes too long to run")
82
def test_llama4_nvfp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
83
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP4", "1")
84
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")
83
- can_initialize("nvidia/Llama-4-Scout-17B-16E-Instruct-FP4", [])
85
+ can_initialize("nvidia/Llama-4-Scout-17B-16E-Instruct-FP4")
86
87
88
@pytest.mark.skip(reason="RuntimeError: No kernel found for the given options")
89
def test_llama4_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
90
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP4", "1")
91
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "latency")
90
- can_initialize("nvidia/Llama-4-Scout-17B-16E-Instruct-FP4", [])
92
+ can_initialize("nvidia/Llama-4-Scout-17B-16E-Instruct-FP4")
93
94
95
## DeepSeekV3 ##
@@ -95,21 +97,37 @@ def test_llama4_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
97
98
def test_deepseek_fp8_block_moe_deep_gemm(monkeypatch: pytest.MonkeyPatch):
99
monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "1")
98
- can_initialize("deepseek-ai/DeepSeek-V3.1", [])
100
+ can_initialize("deepseek-ai/DeepSeek-V3.1")
101
+
102
+
103
+@pytest.mark.skip(reason=("Known issue: lack of kernel support. "
104
+ "Expected failure: assert self.block_quant is None"))
105
+def test_deepseek_fp8_block_moe_flashinfer_cutlass(
106
+ monkeypatch: pytest.MonkeyPatch):
107
+ monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP8", "1")
108
+ monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")
109
+ can_initialize("deepseek-ai/DeepSeek-V3.1")
110
+
111
+
112
+def test_deepseek_fp8_block_moe_flashinfer_trtllm(
113
+ monkeypatch: pytest.MonkeyPatch):
114
+ monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP8", "1")
115
+ monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "latency")
116
+ can_initialize("deepseek-ai/DeepSeek-V3.1")
117
118
119
def test_deepseek_nvfp4_moe_flashinfer_cutlass(
120
monkeypatch: pytest.MonkeyPatch):
121
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP4", "1")
122
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")
105
- can_initialize("nvidia/DeepSeek-R1-0528-FP4-v2", [])
123
+ can_initialize("nvidia/DeepSeek-R1-0528-FP4-v2")
124
125
126
@pytest.mark.skip(reason="RuntimeError: No kernel found for the given options")
127
def test_deepseek_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
128
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP4", "1")
129
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "latency")
112
- can_initialize("nvidia/DeepSeek-R1-0528-FP4-v2", [])
130
+ can_initialize("nvidia/DeepSeek-R1-0528-FP4-v2")
131
132
133
## GPT-OSS ##
@@ -117,16 +135,16 @@ def test_deepseek_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
135
136
def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch):
137
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", "1")
120
- can_initialize("openai/gpt-oss-20b", [])
138
+ can_initialize("openai/gpt-oss-20b")
139
140
141
def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(
142
monkeypatch: pytest.MonkeyPatch):
143
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", "1")
126
- can_initialize("openai/gpt-oss-20b", [])
144
+ can_initialize("openai/gpt-oss-20b")
145
146
147
def test_gptoss_mxfp4mxfp8_moe_flashinfer_trtllm(
148
monkeypatch: pytest.MonkeyPatch):
149
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "1")
132
- can_initialize("openai/gpt-oss-20b", [])
150
+ can_initialize("openai/gpt-oss-20b")