A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Misc] add group_size is -1 in awq quantization (#18910)
Signed-off-by: rongfu.leng <rongfu.leng@daocloud.io>
rongfu.leng committed
May 31, 2025 at 01:34 UTC
7f21e8052b5f3948c8a59514a8dc1e9c5eef70d6
1 file changed
+11
-3
vllm/model_executor/layers/quantization/awq.py
+11
-3
@@ -101,7 +101,13 @@ class AWQLinearMethod(LinearMethodBase):
101
output_partition_sizes: list[int], input_size: int,
102
output_size: int, params_dtype: torch.dtype,
103
**extra_weight_attrs):
104
- if input_size_per_partition % self.quant_config.group_size != 0:
104
+ # Normalize group_size
105
+ if self.quant_config.group_size != -1:
106
+ group_size = self.quant_config.group_size
107
+ else:
108
+ group_size = input_size
109
+
110
+ if input_size_per_partition % group_size != 0:
111
raise ValueError(
112
"The input size is not aligned with the quantized "
113
"weight shape. This can be caused by too large "
@@ -127,9 +133,11 @@ class AWQLinearMethod(LinearMethodBase):
133
packed_factor=self.quant_config.pack_factor,
134
weight_loader=weight_loader)
135
136
+ num_groups = input_size_per_partition // group_size
137
+
138
qzeros = PackedvLLMParameter(
139
data=torch.empty(
132
- input_size_per_partition // self.quant_config.group_size,
140
+ num_groups,
141
output_size_per_partition // self.quant_config.pack_factor,
142
dtype=torch.int32,
143
),
@@ -140,7 +148,7 @@ class AWQLinearMethod(LinearMethodBase):
148
weight_loader=weight_loader)
149
150
scales = GroupQuantScaleParameter(data=torch.empty(
143
- input_size_per_partition // self.quant_config.group_size,
151
+ num_groups,
152
output_size_per_partition,
153
dtype=params_dtype,
154
),