A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Misc] Update `qqq` to use vLLMParameters (#7805)
Dipika Sikka committed
Aug 26, 2024 at 15:16 UTC
665304092de6d56aaccaadacfa497a7836d88e7b
3 files changed
+55
-65
tests/weight_loading/models.txt
+3
-1
@@ -17,4 +17,6 @@ awq, casperhansen/mixtral-instruct-awq, main
17
awq_marlin, casperhansen/mixtral-instruct-awq, main
18
fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
19
marlin, nm-testing/zephyr-beta-7b-marlin-g128, main
20
-marlin, robertgshaw2/zephyr-7b-beta-channelwise-marlin, main
\ No newline at end of file
20
+marlin, robertgshaw2/zephyr-7b-beta-channelwise-marlin, main
21
+qqq, HandH1998/QQQ-Llama-3-8b-g128, main
22
+qqq, HandH1998/QQQ-Llama-3-8b, main
\ No newline at end of file
vllm/model_executor/layers/linear.py
+1
-1
@@ -23,7 +23,7 @@ logger = init_logger(__name__)
23
WEIGHT_LOADER_V2_SUPPORTED = [
24
"CompressedTensorsLinearMethod", "AWQMarlinLinearMethod",
25
"AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod",
26
- "MarlinLinearMethod"
26
+ "MarlinLinearMethod", "QQQLinearMethod"
27
]
28
29
vllm/model_executor/layers/quantization/qqq.py
+51
-63
@@ -8,7 +8,10 @@ from vllm.logger import init_logger
8
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9
from vllm.model_executor.layers.quantization.base_config import (
10
QuantizationConfig)
11
-from vllm.model_executor.utils import set_weight_attrs
11
+from vllm.model_executor.parameter import (BasevLLMParameter,
12
+ ChannelQuantScaleParameter,
13
+ GroupQuantScaleParameter,
14
+ PackedvLLMParameter)
15
16
logger = init_logger(__name__)
17
@@ -133,6 +136,7 @@ class QQQLinearMethod(LinearMethodBase):
136
params_dtype: torch.dtype,
137
**extra_weight_attrs,
138
):
139
+ weight_loader = extra_weight_attrs["weight_loader"]
140
if params_dtype != torch.float16:
141
raise ValueError(
142
f"The params dtype must be float16, but got {params_dtype}")
@@ -170,90 +174,74 @@ class QQQLinearMethod(LinearMethodBase):
174
"Each permutation group must reside on the same gpu")
175
176
# Quantized 4Bit weights packed into Int32.
173
- qweight = Parameter(
174
- torch.empty(
177
+ qweight = PackedvLLMParameter(
178
+ data=torch.empty(
179
input_size_per_partition // self.quant_config.tile_size,
180
output_size_per_partition * self.quant_config.tile_size //
181
self.quant_config.pack_factor,
182
device="cuda",
183
dtype=torch.int32,
184
),
181
- requires_grad=False,
182
- )
183
- set_weight_attrs(
184
- qweight,
185
- {
186
- "input_dim": 0,
187
- "output_dim": 1,
188
- "packed_dim": 1,
189
- "pack_factor": self.quant_config.pack_factor,
190
- "marlin_tile_size": self.quant_config.tile_size,
191
- },
192
- )
193
-
194
- s_channel = Parameter(
195
- torch.empty(
196
- 1,
197
- output_size_per_partition,
198
- device="cuda",
199
- dtype=torch.float,
200
- ),
201
- requires_grad=False,
202
- )
203
- set_weight_attrs(
204
- s_channel,
205
- {
206
- "input_dim": None,
207
- "output_dim": 1,
208
- },
209
- )
185
+ input_dim=0,
186
+ output_dim=1,
187
+ packed_dim=1,
188
+ packed_factor=self.quant_config.pack_factor,
189
+ marlin_tile_size=self.quant_config.tile_size,
190
+ weight_loader=weight_loader)
191
+
192
+ s_channel = ChannelQuantScaleParameter(data=torch.empty(
193
+ 1,
194
+ output_size_per_partition,
195
+ device="cuda",
196
+ dtype=torch.float,
197
+ ),
198
+ weight_loader=weight_loader,
199
+ output_dim=1)
200
201
if self.quant_config.group_size == -1:
212
- s_group = Parameter(
213
- torch.tensor(
214
- [],
215
- device="cuda",
216
- dtype=torch.half,
217
- ),
218
- requires_grad=False,
202
+ s_group_data = torch.tensor(
203
+ [],
204
+ device="cuda",
205
+ dtype=torch.half,
206
)
207
else:
221
- s_group = Parameter(
222
- torch.empty(
223
- input_size_per_partition // self.quant_config.group_size,
224
- output_size_per_partition,
225
- device="cuda",
226
- dtype=torch.half,
227
- ),
228
- requires_grad=False,
208
+ s_group_data = torch.empty(
209
+ input_size_per_partition // self.quant_config.group_size,
210
+ output_size_per_partition,
211
+ device="cuda",
212
+ dtype=torch.half,
213
)
214
231
- set_weight_attrs(
232
- s_group,
233
- {
234
- "input_dim": None if self.quant_config.group_size == -1 else 0,
235
- "output_dim":
236
- None if self.quant_config.group_size == -1 else 1,
237
- },
238
- )
215
+ s_group_attr = {"data": s_group_data, "weight_loader": weight_loader}
216
+
217
+ if self.quant_config.group_size == -1:
218
+ s_group = BasevLLMParameter(**s_group_attr)
219
+ else:
220
+ s_group = GroupQuantScaleParameter(output_dim=1,
221
+ input_dim=0,
222
+ **s_group_attr)
223
224
# Allocate workspace (Used for internal locking mechanism)
225
max_workspace_size = (
226
output_size_per_partition //
227
self.quant_config.min_n_threads) * self.quant_config.max_parallel
244
- workspace = Parameter(torch.zeros(max_workspace_size,
245
- device="cuda",
246
- dtype=torch.int),
247
- requires_grad=False)
228
+
229
+ workspace = BasevLLMParameter(data=torch.zeros(max_workspace_size,
230
+ device="cuda",
231
+ dtype=torch.int),
232
+ weight_loader=weight_loader)
233
234
layer.register_parameter("B", qweight)
250
- set_weight_attrs(qweight, extra_weight_attrs)
235
layer.register_parameter("s_channel", s_channel)
252
- set_weight_attrs(s_channel, extra_weight_attrs)
236
layer.register_parameter("s_group", s_group)
254
- set_weight_attrs(s_group, extra_weight_attrs)
237
layer.register_parameter("workspace", workspace)
256
- set_weight_attrs(workspace, extra_weight_attrs)
238
+
239
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
240
+ # required by torch.compile
241
+ layer.B = Parameter(layer.B.data, requires_grad=False)
242
+ layer.s_channel = Parameter(layer.s_channel.data, requires_grad=False)
243
+ layer.s_group = Parameter(layer.s_group.data, requires_grad=False)
244
+ layer.workspace = Parameter(layer.workspace.data, requires_grad=False)
245
246
def apply(
247
self,