A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bug] Fix Weight Loading for Block FP8 Cutlass SM90 (#25909)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Wentao Ye committed
Sep 29, 2025 at 21:15 UTC
89e4050af4299f461077ed6c4dfa1318693d40ee
1 file changed
+4
-4
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+4
-4
@@ -911,15 +911,15 @@ def maybe_post_process_fp8_weight_block(layer: torch.nn.Module,
911
# On Blackwell or Hopper, if E8M0 for DeepGemm is used, we need to
912
# requantize the weight and input to the specific scale
913
# at the same time.
914
- if is_deep_gemm_e8m0_used():
914
+ should_use_deepgemm = should_use_deepgemm_for_fp8_linear(
915
+ layer.orig_dtype, layer.weight)
916
+ if is_deep_gemm_e8m0_used() and should_use_deepgemm:
917
block_sz = tuple(layer.weight_block_size)
918
requant_weight_ue8m0_inplace(layer.weight.data,
919
layer.weight_scale.data, block_sz)
920
# SM90 Block FP8 CUTLASS requires row-major weight scales
921
elif (current_platform.is_device_capability(90)
920
- and cutlass_block_fp8_supported
921
- and not should_use_deepgemm_for_fp8_linear(torch.bfloat16,
922
- layer.weight)):
922
+ and cutlass_block_fp8_supported and not should_use_deepgemm):
923
layer.weight_scale = torch.nn.Parameter(
924
layer.weight_scale.data.T.contiguous(), requires_grad=False)
925