A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Misc] Update `gptq_marlin_24` to use vLLMParameters (#7762)
Co-authored-by: Michael Goin <michael@neuralmagic.com>
Dipika Sikka committed
Aug 26, 2024 at 17:44 UTC
dd9857f5fae74d2fd8fc236419f736a4663db800
2 files changed
+50
-54
vllm/model_executor/layers/linear.py
+1
-1
@@ -23,7 +23,7 @@ logger = init_logger(__name__)
23
WEIGHT_LOADER_V2_SUPPORTED = [
24
"CompressedTensorsLinearMethod", "AWQMarlinLinearMethod",
25
"AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod",
26
- "MarlinLinearMethod", "QQQLinearMethod"
26
+ "MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod"
27
]
28
29
vllm/model_executor/layers/quantization/gptq_marlin_24.py
+49
-53
@@ -8,7 +8,10 @@ from vllm.logger import init_logger
8
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9
from vllm.model_executor.layers.quantization.base_config import (
10
QuantizationConfig)
11
-from vllm.model_executor.utils import set_weight_attrs
11
+from vllm.model_executor.parameter import (BasevLLMParameter,
12
+ ChannelQuantScaleParameter,
13
+ GroupQuantScaleParameter,
14
+ PackedvLLMParameter)
15
from vllm.scalar_type import scalar_types
16
17
logger = init_logger(__name__)
@@ -149,7 +152,7 @@ class GPTQMarlin24LinearMethod(LinearMethodBase):
152
**extra_weight_attrs,
153
):
154
del output_size # Unused.
152
-
155
+ weight_loader = extra_weight_attrs["weight_loader"]
156
if params_dtype != torch.float16:
157
raise ValueError(
158
f"The params dtype must be float16, but got {params_dtype}")
@@ -187,87 +190,80 @@ class GPTQMarlin24LinearMethod(LinearMethodBase):
190
"Each permutation group must reside on the same gpu")
191
192
# Quantized 4Bit weights packed into Int32.
190
- qweight = Parameter(
191
- torch.empty(
193
+ qweight = PackedvLLMParameter(
194
+ data=torch.empty(
195
input_size_per_partition // self.quant_config.tile_size // 2,
196
output_size_per_partition * self.quant_config.tile_size //
197
self.quant_config.pack_factor,
198
device="cuda",
199
dtype=torch.int32,
200
),
198
- requires_grad=False,
199
- )
200
- set_weight_attrs(
201
- qweight,
202
- {
203
- "input_dim": 0,
204
- "output_dim": 1,
205
- "packed_dim": 1,
206
- "pack_factor": self.quant_config.pack_factor,
207
- "marlin_tile_size": self.quant_config.tile_size,
208
- },
209
- )
201
+ input_dim=0,
202
+ output_dim=1,
203
+ packed_dim=1,
204
+ packed_factor=self.quant_config.pack_factor,
205
+ marlin_tile_size=self.quant_config.tile_size,
206
+ weight_loader=weight_loader)
207
208
# Meta
212
- meta = Parameter(
213
- torch.empty(
214
- input_size_per_partition // 8 // 2 // 2,
215
- output_size_per_partition * 2,
216
- device="cuda",
217
- dtype=torch.int16,
218
- ),
219
- requires_grad=False,
220
- )
221
- set_weight_attrs(
222
- meta,
223
- {
224
- "input_dim": 0,
225
- "packed_dim": 1,
226
- "pack_factor": 1,
227
- "output_dim": 1,
228
- "marlin_tile_size": 2,
229
- },
230
- )
209
+ meta = PackedvLLMParameter(data=torch.empty(
210
+ input_size_per_partition // 8 // 2 // 2,
211
+ output_size_per_partition * 2,
212
+ device="cuda",
213
+ dtype=torch.int16,
214
+ ),
215
+ input_dim=0,
216
+ output_dim=1,
217
+ packed_dim=1,
218
+ packed_factor=1,
219
+ marlin_tile_size=2,
220
+ weight_loader=weight_loader)
221
222
# Determine if channelwise or not
223
input_groups = (1 if self.quant_config.group_size == -1 else
224
input_size_per_partition //
225
self.quant_config.group_size)
226
237
- scales = Parameter(
227
+ weight_scale_args = {
228
+ "data":
229
torch.empty(
230
input_groups,
231
output_size_per_partition,
232
device="cuda",
233
dtype=params_dtype,
234
),
244
- requires_grad=False,
245
- )
246
- set_weight_attrs(
247
- scales,
248
- {
249
- "input_dim": None if input_groups == 1 else 0,
250
- "output_dim": 1,
251
- },
252
- )
235
+ "weight_loader":
236
+ weight_loader
237
+ }
238
+ if input_groups == 1:
239
+ scales = ChannelQuantScaleParameter(output_dim=1,
240
+ **weight_scale_args)
241
+ else:
242
+ scales = GroupQuantScaleParameter(output_dim=1,
243
+ input_dim=0,
244
+ **weight_scale_args)
245
246
# Allocate workspace (Used for internal locking mechanism)
247
max_workspace_size = (
248
output_size_per_partition //
249
self.quant_config.min_n_threads) * self.quant_config.max_parallel
258
- workspace = Parameter(torch.zeros(max_workspace_size,
259
- device="cuda",
260
- dtype=torch.int),
261
- requires_grad=False)
250
+
251
+ workspace = BasevLLMParameter(data=torch.zeros(max_workspace_size,
252
+ device="cuda",
253
+ dtype=torch.int),
254
+ weight_loader=weight_loader)
255
256
layer.register_parameter("B_24", qweight)
264
- set_weight_attrs(qweight, extra_weight_attrs)
257
layer.register_parameter("B_meta", meta)
266
- set_weight_attrs(meta, extra_weight_attrs)
258
layer.register_parameter("s", scales)
268
- set_weight_attrs(scales, extra_weight_attrs)
259
layer.register_parameter("workspace", workspace)
270
- set_weight_attrs(workspace, extra_weight_attrs)
260
+
261
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
262
+ # required by torch.compile
263
+ layer.B_24 = Parameter(layer.B_24.data, requires_grad=False)
264
+ layer.s = Parameter(layer.s.data, requires_grad=False)
265
+ layer.B_meta = Parameter(layer.B_meta.data, requires_grad=False)
266
+ layer.workspace = Parameter(layer.workspace.data, requires_grad=False)
267
268
def apply(
269
self,