A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 11:00 UTC
[Bugfix] Fix marlin nvfp4 rescaling (#37502)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
Jinzhen Lin committed
Apr 7, 2026 at 23:57 UTC
731055548262317579c35189a46cde6932fcc376
1 file changed
+6
-4
vllm/model_executor/layers/quantization/utils/marlin_utils_fp4.py
+6
-4
@@ -43,9 +43,9 @@ def _nvfp4_compute_scale_factor(
43
ws_float = marlin_scales.float() * (2**7)
44
nonzero_mask = ws_float > 0
45
if nonzero_mask.any():
46
- min_val = ws_float[nonzero_mask].min()
47
- if min_val < 2:
48
- sf = (2 / min_val).log2().ceil().exp2()
46
+ max_val = ws_float[nonzero_mask].max()
47
+ if max_val < 448 * (2**7):
48
+ sf = (448 * (2**7) / max_val).log2().floor().exp2()
49
return sf.item()
50
return 1.0
51
@@ -105,7 +105,9 @@ def nvfp4_marlin_process_scales(
105
if scale_factor > 1.0:
106
marlin_scales = (marlin_scales.float() * scale_factor).to(torch.half)
107
108
- marlin_scales = (marlin_scales * (2**7)).view(torch.int16) << 1
108
+ marlin_scales = marlin_scales * (2**7)
109
+ marlin_scales[marlin_scales < 2] = 0
110
+ marlin_scales = marlin_scales.view(torch.int16) << 1
111
marlin_scales = marlin_scales.view(torch.float8_e4m3fn)
112
marlin_scales = marlin_scales[:, 1::2].contiguous()
113