A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Perf] Fix DeepGEMM Contiguous Layout Issue, 5.5% Throughput Improvement (#24783)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Wentao Ye committed
Sep 14, 2025 at 11:20 UTC
fc2dbcda8b717e6aac0794cb9b4cc86b78c36504
1 file changed
+4
-4
vllm/model_executor/layers/quantization/fp8.py
+4
-4
@@ -772,10 +772,10 @@ class Fp8MoEMethod(FusedMoEMethodBase):
772
if self.allow_deep_gemm and not is_deep_gemm_e8m0_used():
773
if _is_col_major(layer.w13_weight_scale_inv):
774
layer.w13_weight_scale_inv = \
775
- get_col_major_tma_aligned_tensor(layer.w13_weight_scale_inv).contiguous()
775
+ get_col_major_tma_aligned_tensor(layer.w13_weight_scale_inv)
776
if _is_col_major(layer.w2_weight_scale_inv):
777
layer.w2_weight_scale_inv = \
778
- get_col_major_tma_aligned_tensor(layer.w2_weight_scale_inv).contiguous()
778
+ get_col_major_tma_aligned_tensor(layer.w2_weight_scale_inv)
779
780
# If checkpoint is fp16, quantize in place.
781
elif not self.quant_config.is_checkpoint_fp8_serialized:
@@ -923,10 +923,10 @@ class Fp8MoEMethod(FusedMoEMethodBase):
923
# Ensure column-major TMA alignment expected by DeepGEMM.
924
if _is_col_major(layer.w13_weight_scale_inv):
925
layer.w13_weight_scale_inv = get_col_major_tma_aligned_tensor(
926
- layer.w13_weight_scale_inv).contiguous()
926
+ layer.w13_weight_scale_inv)
927
if _is_col_major(layer.w2_weight_scale_inv):
928
layer.w2_weight_scale_inv = get_col_major_tma_aligned_tensor(
929
- layer.w2_weight_scale_inv).contiguous()
929
+ layer.w2_weight_scale_inv)
930
931
def select_gemm_impl(
932
self,