A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Bugfix] Remove NVFP4 scales assertions to fix load_format=dummy (#18861)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
May 30, 2025 at 01:37 UTC
4d0a1541bed0a89f71c55b0714e18422688bec53
2 files changed
+11
-4
vllm/model_executor/layers/quantization/modelopt.py
+5
-3
@@ -585,9 +585,11 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
585
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
586
587
# GEMM 1
588
- assert torch.allclose(
589
- layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]), (
590
- "w1_weight_scale_2 must match w3_weight_scale_2")
588
+ if not torch.allclose(layer.w13_weight_scale_2[:, 0],
589
+ layer.w13_weight_scale_2[:, 1]):
590
+ logger.warning_once(
591
+ "w1_weight_scale_2 must match w3_weight_scale_2. "
592
+ "Accuracy may be affected.")
593
594
w13_weight_scale_2 = layer.w13_weight_scale_2[:, 0]
595
layer.w13_weight_scale_2 = Parameter(w13_weight_scale_2,
vllm/model_executor/layers/quantization/utils/marlin_utils_fp4.py
+6
-1
@@ -22,7 +22,12 @@ def is_fp4_marlin_supported():
22
23
24
def fp4_marlin_process_scales(marlin_scales):
25
- assert (marlin_scales >= 0).all()
25
+ if not (marlin_scales >= 0).all():
26
+ logger.warning_once(
27
+ "NVFP4 Marlin assumes the scales to be >=0, but has encountered "
28
+ "negative scales. Accuracy will likely be degraded. This is "
29
+ "because it changes the scales from FP8-S1E4M3 to a special "
30
+ "FP8-S0E5M3 format to speedup the dequantization.")
31
32
# convert to half first, we would convert to fp8 later
33
marlin_scales = marlin_scales.to(torch.half)