A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 11:30 UTC
[Bugfix] Fix vllm serve failure with Nemotron Nano V3 FP8 (#31960)
Signed-off-by: Daniel Serebrenik <daserebrenik@nvidia.com>
danisereb committed
Jan 8, 2026 at 18:08 UTC
b8112c1d85442b060e37df90e9db343cbc7b000c
1 file changed
+4
-3
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+4
-3
@@ -1619,11 +1619,12 @@ def process_fp8_weight_tensor_strategy_moe(
1619
"""Process moe weights for tensor-wise quantization strategy."""
1620
max_scales = weight_scales.max(dim=1).values
1621
1622
- # For w1 case (i.e. not w13): just collapse the last dim since
1623
- # there is already just one scale per expert in this case.
1622
+ # For w1 case (i.e. not w13): there is already just one scale per expert.
1623
if not is_act_and_mul:
1624
assert weight_scales.shape[1] == 1
1626
- return weight, weight_scales.max()
1625
+ # One scale per expert
1626
+ assert max_scales.shape == (num_experts,)
1627
+ return weight, max_scales
1628
1629
# For w13 case (common): require single scale for w13 per expert, but
1630
# on disk there is a scale for w1 and w3. Use the max to requantize.