A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BugFix] Skip the Q component for QKVParallelLinear in the case of QKVCrossParallelLinear since its width is 0 (#22369)
Signed-off-by: sstamenk <sstamenk@amd.com>
sstamenk committed
Aug 15, 2025 at 19:17 UTC
6b04039a7240ae1039fea4bd179ec3b452f19107
1 file changed
+3
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+3
@@ -121,6 +121,9 @@ def requantize_with_max_scale(
121
if unfused_module_in_checkpoint:
122
start = 0
123
for idx, logical_width in enumerate(logical_widths):
124
+ # Skip any component with zero width.
125
+ if logical_width == 0:
126
+ continue
127
end = start + logical_width
128
weight_dq = per_tensor_dequantize(weight[start:end, :],
129
weight_scale[idx])