A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 21:15 UTC
[Bugfix] Fix W4A8_FP8 MoE tp>1 correctness and view() TypeError (#40310)
Signed-off-by: EdalatiAli <aliedalati@cohere.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
EdalatiAli committed
Apr 21, 2026 at 21:58 UTC
f946659fff3c4be2105a4ebfdd4dd0dbec0d6b8a
2 files changed
+5
-1
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py
+4
@@ -198,11 +198,15 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
198
# encode and reorder weight tensors, and get the layout to pass to
199
# the grouped gemm kernel. `b_strides1/2` specifies the entire layout
200
convert_packed_uint4b8_to_signed_int4_inplace(layer.w13_weight_packed)
201
+ # mirror the sync in CutlassW4A8LinearKernel; required for tp>1 correctness
202
+ torch.accelerator.synchronize()
203
w13_weight_shuffled, self.b_strides1 = (
204
ops.cutlass_encode_and_reorder_int4b_grouped(layer.w13_weight_packed)
205
)
206
replace_parameter(layer, "w13_weight_packed", w13_weight_shuffled)
207
convert_packed_uint4b8_to_signed_int4_inplace(layer.w2_weight_packed)
208
+ # mirror the sync in CutlassW4A8LinearKernel; required for tp>1 correctness
209
+ torch.accelerator.synchronize()
210
w2_weight_shuffled, self.b_strides2 = (
211
ops.cutlass_encode_and_reorder_int4b_grouped(layer.w2_weight_packed)
212
)
vllm/model_executor/layers/quantization/utils/quant_utils.py
+1
-1
@@ -818,7 +818,7 @@ def convert_bf16_scales_to_fp8(
818
819
# restore original shape
820
fp8_scales = fp8_scales.view(orig_shape)
821
- chan_scales = chan_scales.view(orig_shape[:-1], -1)
821
+ chan_scales = chan_scales.view(*orig_shape[:-1], -1)
822
823
return fp8_scales, chan_scales
824