A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 23:30 UTC
[Bugfix] Fix DeepGemm E8M0 accuracy degradation for Qwen3.5 FP8 on Blackwell (#38083)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
Vadim Gimpelson committed
Mar 26, 2026 at 12:21 UTC
52069012fe53714581682ded6746cd311108bf3a
10 files changed
+69
-11
tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-DEP2.yaml
+2
-1
@@ -1,5 +1,6 @@
1
model_name: "Qwen/Qwen3.5-35B-A3B"
2
-accuracy_threshold: 0.86
2
+accuracy_threshold: 0.84
3
+tolerance: 0.03
4
num_questions: 1319
5
num_fewshot: 5
6
server_args: >-
tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-FP8-DEP2.yaml
+2
-1
@@ -1,5 +1,6 @@
1
model_name: "Qwen/Qwen3.5-35B-A3B-FP8"
2
-accuracy_threshold: 0.86
2
+accuracy_threshold: 0.79
3
+tolerance: 0.03
4
num_questions: 1319
5
num_fewshot: 5
6
server_args: >-
tests/evals/gsm8k/configs/Qwen3.5-397B-A17B-NVFP4-DEP2.yaml
new
+9
@@ -0,0 +1,9 @@
1
+model_name: "nvidia/Qwen3.5-397B-A17B-NVFP4"
2
+accuracy_threshold: 0.88
3
+tolerance: 0.03
4
+num_questions: 1319
5
+num_fewshot: 5
6
+server_args: >-
7
+ --max-model-len 4096
8
+ --data-parallel-size 2
9
+ --enable-expert-parallel
tests/evals/gsm8k/configs/models-qwen35-blackwell.txt
+2
@@ -1 +1,3 @@
1
Qwen3.5-35B-A3B-DEP2.yaml
2
+Qwen3.5-35B-A3B-FP8-DEP2.yaml
3
+Qwen3.5-397B-A17B-NVFP4-DEP2.yaml
\ No newline at end of file
tests/evals/gsm8k/test_gsm8k_correctness.py
+4
-6
@@ -19,8 +19,6 @@ from vllm.platforms import current_platform
19
20
from .gsm8k_eval import evaluate_gsm8k
21
22
-TOL = 0.08 # Absolute tolerance for accuracy comparison
23
-
22
23
def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
24
"""Run GSM8K evaluation using our isolated script."""
@@ -109,20 +107,20 @@ def test_gsm8k_correctness(config_filename):
107
108
measured_metric = results["accuracy"]
109
expected_metric = eval_config["accuracy_threshold"]
110
+ tol = eval_config.get("tolerance", 0.08)
111
112
print(f"GSM8K Results for {eval_config['model_name']}:")
113
print(f" Measured metric: {measured_metric:.4f}")
114
print(f" Expected metric: {expected_metric:.4f}")
116
- print(f" Tolerance: {TOL:.4f}")
115
+ print(f" Tolerance: {tol:.4f}")
116
print(f" Questions: {results['num_questions']}")
117
print(f" Invalid rate: {results['invalid_rate']:.3f}")
118
print(f" Latency: {results['latency']:.1f}s")
119
print(f" QPS: {results['questions_per_second']:.1f}")
120
122
- # Verify metric is within tolerance
123
- assert measured_metric >= expected_metric - TOL, (
121
+ assert measured_metric >= expected_metric - tol, (
122
f"GSM8K metric too low: {measured_metric:.4f} < "
125
- f"{expected_metric:.4f} - {TOL:.4f} = {expected_metric - TOL:.4f}"
123
+ f"{expected_metric:.4f} - {tol:.4f} = {expected_metric - tol:.4f}"
124
)
125
126
print(f"✅ GSM8K test passed for {eval_config['model_name']}")
vllm/config/vllm.py
+19
@@ -701,6 +701,25 @@ class VllmConfig:
701
self.model_config, self.load_config
702
)
703
704
+ if (
705
+ self.quant_config is not None
706
+ and self.model_config is not None
707
+ and hasattr(self.quant_config, "use_deep_gemm")
708
+ and self.quant_config.use_deep_gemm is None
709
+ ):
710
+ from vllm.utils.deep_gemm import should_auto_disable_deep_gemm
711
+
712
+ model_type = getattr(self.model_config.hf_text_config, "model_type", None)
713
+ if should_auto_disable_deep_gemm(model_type):
714
+ self.quant_config.use_deep_gemm = False
715
+ logger.warning_once(
716
+ "Auto-disabled DeepGemm for model_type=%s on Blackwell. "
717
+ "DeepGemm E8M0 scale format causes accuracy degradation "
718
+ "for this architecture. Falling back to CUTLASS. "
719
+ "To disable DeepGemm globally, set VLLM_USE_DEEP_GEMM=0.",
720
+ model_type,
721
+ )
722
+
723
from vllm.v1.executor.abstract import Executor
724
725
executor_backend = self.parallel_config.distributed_executor_backend
vllm/model_executor/layers/quantization/fp8.py
+7
-2
@@ -129,6 +129,7 @@ class Fp8Config(QuantizationConfig):
129
f"{activation_scheme} activation scheme."
130
)
131
self.weight_block_size = weight_block_size
132
+ self.use_deep_gemm: bool | None = None
133
134
@classmethod
135
def get_name(cls) -> QuantizationMethods:
@@ -276,7 +277,10 @@ class Fp8LinearMethod(LinearMethodBase):
277
self.marlin_input_dtype = None
278
279
self.use_aiter_and_is_supported = rocm_aiter_ops.is_linear_fp8_enabled()
279
- self.use_deep_gemm = is_deep_gemm_supported()
280
+ if self.quant_config.use_deep_gemm is not None:
281
+ self.use_deep_gemm = self.quant_config.use_deep_gemm
282
+ else:
283
+ self.use_deep_gemm = is_deep_gemm_supported()
284
285
self.weight_block_size = self.quant_config.weight_block_size
286
self.block_quant = self.weight_block_size is not None
@@ -311,6 +315,7 @@ class Fp8LinearMethod(LinearMethodBase):
315
act_quant_group_shape=GroupShape(1, self.weight_block_size[0]),
316
cutlass_block_fp8_supported=self.cutlass_block_fp8_supported,
317
use_aiter_and_is_supported=self.use_aiter_and_is_supported,
318
+ use_deep_gemm=self.use_deep_gemm,
319
)
320
321
def create_weights(
@@ -428,7 +433,7 @@ class Fp8LinearMethod(LinearMethodBase):
433
else:
434
layer.input_scale = None
435
431
- if self.block_quant:
436
+ if self.block_quant and self.use_deep_gemm:
437
maybe_post_process_fp8_weight_block(layer)
438
439
def apply(
vllm/model_executor/layers/quantization/input_quant_fp8.py
+1
@@ -91,6 +91,7 @@ class QuantFP8(CustomOp):
91
92
if (
93
self.is_group_quant
94
+ and self.use_ue8m0
95
and self.use_deep_gemm_supported
96
and (DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0)
97
):
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+5
-1
@@ -359,10 +359,14 @@ class W8A8BlockFp8LinearOp:
359
act_quant_group_shape: GroupShape,
360
cutlass_block_fp8_supported: bool = CUTLASS_BLOCK_FP8_SUPPORTED,
361
use_aiter_and_is_supported: bool = False,
362
+ use_deep_gemm: bool | None = None,
363
):
364
self.weight_group_shape = weight_group_shape
365
self.act_quant_group_shape = act_quant_group_shape
365
- self.is_deep_gemm_supported = is_deep_gemm_supported()
366
+ if use_deep_gemm is not None:
367
+ self.is_deep_gemm_supported = use_deep_gemm
368
+ else:
369
+ self.is_deep_gemm_supported = is_deep_gemm_supported()
370
self.is_hopper = current_platform.is_device_capability(90)
371
self.use_deep_gemm_e8m0 = is_deep_gemm_e8m0_used()
372
self.is_flashinfer_supported = is_flashinfer_fp8_blockscale_gemm_supported()
vllm/utils/deep_gemm.py
+18
@@ -23,6 +23,24 @@ from vllm.platforms import current_platform
23
from vllm.utils.import_utils import has_deep_gemm
24
from vllm.utils.math_utils import cdiv
25
26
+_DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES: set[str] = {
27
+ "qwen3_5_text",
28
+ "qwen3_5_moe_text",
29
+}
30
+
31
+
32
+def should_auto_disable_deep_gemm(model_type: str | None) -> bool:
33
+ """Check if DeepGemm should be auto-disabled for this model on Blackwell.
34
+
35
+ Returns True if the model is known to have accuracy degradation with
36
+ DeepGemm's E8M0 scale format on Blackwell GPUs (SM100+).
37
+ """
38
+ if model_type is None:
39
+ return False
40
+ if not current_platform.is_device_capability_family(100):
41
+ return False
42
+ return model_type in _DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES
43
+
44
45
class DeepGemmQuantScaleFMT(Enum):
46
# Float32 scales in Float32 tensor