A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bug] Fix Layer `weight_block_size` Assertion Issue (#24674)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Sep 11, 2025 at 19:47 UTC
fcba05c4353c65d93b5299b50aa6b475e467c01a
1 file changed
+3
-3
vllm/model_executor/layers/quantization/fp8.py
+3
-3
@@ -450,10 +450,10 @@ class Fp8LinearMethod(LinearMethodBase):
450
# Activations not quantized for marlin.
451
del layer.input_scale
452
453
- # On B200, if E8M0 for DeepGemm is used, we need to
453
+ # On Blackwell or Hopper, if E8M0 for DeepGemm is used, we need to
454
# requantize the weight and input to the specific scale
455
# at the same time.
456
- if is_deep_gemm_e8m0_used():
456
+ if is_deep_gemm_e8m0_used() and self.block_quant:
457
assert layer.weight_block_size is not None
458
block_sz = tuple(layer.weight_block_size)
459
requant_weight_ue8m0_inplace(
@@ -905,7 +905,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
905
del layer.w13_input_scale
906
del layer.w2_input_scale
907
908
- if is_deep_gemm_e8m0_used():
908
+ if is_deep_gemm_e8m0_used() and self.block_quant:
909
assert layer.weight_block_size is not None
910
# Re-quantise the expert weights so their scales are UE8M0.
911
block_sz = tuple(layer.weight_block_size)