A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 06:15 UTC
[BugFix] Add synchronize in CutlassW4A8LinearKernel to ensure data is ready for use. (#33078)
Co-authored-by: jinwuguo <jinwuguo@tencent.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Jinwu committed
Jan 31, 2026 at 00:14 UTC
f68e3ea4e135c41b997d8226186509659b3d722c
1 file changed
+1
vllm/model_executor/layers/quantization/kernels/mixed_precision/cutlass.py
+1
@@ -77,6 +77,7 @@ class CutlassW4A8LinearKernel(MPLinearKernel):
77
def transform_w_q(x):
78
assert isinstance(x, BasevLLMParameter)
79
convert_packed_uint4b8_to_signed_int4_inplace(x.data)
80
+ torch.cuda.synchronize()
81
permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
82
x.data = ops.cutlass_encode_and_reorder_int4b(x.data.t().contiguous().t())
83
return x