A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Misc] Update `gptq_marlin` to use new vLLMParameters (#7281)
Dipika Sikka committed
Aug 13, 2024 at 14:30 UTC
fb377d7e74228d477e270d8a8e53410db29ed755
8 files changed
+225
-89
.buildkite/test-pipeline.yaml
+10
@@ -314,6 +314,16 @@ steps:
314
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
315
- pytest -v -s -x lora/test_long_context.py
316
317
+- label: Weight Loading Multiple GPU Test
318
+ working_dir: "/vllm-workspace/tests"
319
+ num_gpus: 2
320
+ source_file_dependencies:
321
+ - vllm/
322
+ - tests/weight_loading
323
+ commands:
324
+ - bash weight_loading/run_model_weight_loading_test.sh
325
+
326
+
327
##### multi gpus test #####
328
##### A100 test #####
329
tests/weight_loading/models.txt
new
+15
@@ -0,0 +1,15 @@
1
+gptq_marlin, robertgshaw2/zephyr-7b-beta-channelwise-gptq, main
2
+gptq_marlin, TheBloke/Llama-2-7B-GPTQ, main
3
+gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, main
4
+gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit--1g-actorder_True
5
+gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit-32g-actorder_True
6
+gptq_marlin, TechxGenus/gemma-1.1-2b-it-GPTQ, main
7
+compressed-tensors, nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change, main
8
+compressed-tensors, nm-testing/tinyllama-oneshot-w8-channel-a8-tensor, main
9
+compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2, main
10
+compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2, main
11
+compressed-tensors, nm-testing/tinyllama-oneshot-w4a16-group128-v2, main
12
+compressed-tensors, nm-testing/tinyllama-oneshot-w8a16-per-channel, main
13
+compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main
14
+compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
15
+compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
\ No newline at end of file
tests/weight_loading/run_model_weight_loading_test.sh
new
+32
@@ -0,0 +1,32 @@
1
+#!/bin/bash
2
+SUCCESS=0
3
+
4
+IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < "weight_loading/models.txt"
5
+
6
+for MODEL_CONFIG in "${MODEL_CONFIGS[@]}"
7
+do
8
+ LOCAL_SUCCESS=0
9
+ IFS=', ' read -r -a array <<< "$MODEL_CONFIG"
10
+
11
+ echo "=== RUNNING MODEL: $MODEL_CONFIG ==="
12
+
13
+ export QUANTIZATION=${array[0]}
14
+ export MODEL_NAME=${array[1]}
15
+ export REVISION=${array[2]}
16
+ pytest -s weight_loading/test_weight_loading.py || LOCAL_SUCCESS=$?
17
+
18
+ if [[ $LOCAL_SUCCESS == 0 ]]; then
19
+ echo "=== PASSED MODEL: ${MODEL_CONFIG} ==="
20
+ else
21
+ echo "=== FAILED MODEL: ${MODEL_CONFIG} ==="
22
+ fi
23
+
24
+ SUCCESS=$((SUCCESS + LOCAL_SUCCESS))
25
+
26
+done
27
+
28
+if [ "${SUCCESS}" -eq "0" ]; then
29
+ exit 0
30
+else
31
+ exit 1
32
+fi
tests/weight_loading/test_weight_loading.py
new
+20
@@ -0,0 +1,20 @@
1
+import os
2
+
3
+MAX_MODEL_LEN = 1024
4
+MODEL_NAME = os.environ.get("MODEL_NAME",
5
+ "robertgshaw2/zephyr-7b-beta-channelwise-gptq")
6
+REVISION = os.environ.get("REVISION", "main")
7
+QUANTIZATION = os.environ.get("QUANTIZATION", "gptq_marlin")
8
+
9
+
10
+def test_weight_loading(vllm_runner):
11
+ with vllm_runner(model_name=MODEL_NAME,
12
+ revision=REVISION,
13
+ dtype="auto",
14
+ quantization=QUANTIZATION,
15
+ max_model_len=MAX_MODEL_LEN,
16
+ tensor_parallel_size=2) as model:
17
+
18
+ output = model.generate_greedy("Hello world!", max_tokens=20)
19
+ print(output)
20
+ assert output
vllm/model_executor/layers/linear.py
+3
-1
@@ -20,7 +20,9 @@ from vllm.model_executor.utils import set_weight_attrs
20
21
logger = init_logger(__name__)
22
23
-WEIGHT_LOADER_V2_SUPPORTED = ["CompressedTensorsLinearMethod"]
23
+WEIGHT_LOADER_V2_SUPPORTED = [
24
+ "CompressedTensorsLinearMethod", "GPTQMarlinLinearMethod"
25
+]
26
27
28
def adjust_marlin_shard(param, shard_size, shard_offset):
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+1
-1
@@ -105,7 +105,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
105
dtype=params_dtype,
106
)
107
}
108
- if self.group_size == -1:
108
+ if not partition_scales:
109
weight_scale = ChannelQuantScaleParameter(output_dim=0,
110
**weight_scale_args)
111
else:
vllm/model_executor/layers/quantization/gptq_marlin.py
+59
-62
@@ -1,12 +1,11 @@
1
from typing import Any, Dict, List, Optional
2
3
import torch
4
-from torch.nn.parameter import Parameter
4
+from torch.nn import Parameter
5
6
from vllm import _custom_ops as ops
7
from vllm.logger import init_logger
8
-from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9
- set_weight_attrs)
8
+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9
from vllm.model_executor.layers.quantization.base_config import (
10
QuantizationConfig)
11
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
@@ -15,6 +14,11 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
14
marlin_repeat_scales_on_all_ranks, marlin_sort_g_idx, replace_tensor,
15
verify_marlin_supported, verify_marlin_supports_shape)
16
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
17
+from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
18
+ GroupQuantScaleParameter,
19
+ PackedColumnParameter,
20
+ PackedvLLMParameter,
21
+ RowvLLMParameter)
22
from vllm.scalar_type import scalar_types
23
24
logger = init_logger(__name__)
@@ -159,9 +163,11 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
163
params_dtype: torch.dtype,
164
**extra_weight_attrs,
165
) -> None:
166
+
167
del output_size
168
output_size_per_partition = sum(output_partition_sizes)
169
is_row_parallel = input_size != input_size_per_partition
170
+ weight_loader = extra_weight_attrs.get("weight_loader")
171
172
# Normalize group_size
173
if self.quant_config.group_size != -1:
@@ -190,79 +196,66 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
196
scales_and_zp_size = input_size_per_partition // group_size
197
198
# Quantized weights
193
- qweight = Parameter(
194
- torch.empty(
199
+ qweight = PackedvLLMParameter(
200
+ data=torch.empty(
201
input_size_per_partition // self.quant_config.pack_factor,
202
output_size_per_partition,
203
dtype=torch.int32,
204
),
199
- requires_grad=False,
200
- )
201
- set_weight_attrs(
202
- qweight,
203
- {
204
- **extra_weight_attrs,
205
- "input_dim": 0,
206
- "output_dim": 1,
207
- "packed_dim": 0,
208
- "pack_factor": self.quant_config.pack_factor,
209
- },
210
- )
205
+ input_dim=0,
206
+ output_dim=1,
207
+ packed_dim=0,
208
+ packed_factor=self.quant_config.pack_factor,
209
+ weight_loader=weight_loader)
210
211
# Activation order
213
- g_idx = Parameter(
212
+ g_idx = RowvLLMParameter(data=torch.empty(
213
+ input_size_per_partition,
214
+ dtype=torch.int32,
215
+ ),
216
+ input_dim=0,
217
+ weight_loader=weight_loader)
218
+
219
+ qzeros_args = {
220
+ "data":
221
torch.empty(
215
- input_size_per_partition,
222
+ scales_and_zp_size,
223
+ output_size_per_partition // self.quant_config.pack_factor,
224
dtype=torch.int32,
225
),
218
- requires_grad=False,
219
- )
220
- # Ignore warning from fused linear layers such as QKVParallelLinear.
221
- set_weight_attrs(
222
- g_idx,
223
- {
224
- **extra_weight_attrs, "input_dim": 0,
225
- "ignore_warning": True
226
- },
227
- )
228
-
229
- # Scales
230
- scales = Parameter(
226
+ "weight_loader":
227
+ weight_loader
228
+ }
229
+ weight_scale_args = {
230
+ "data":
231
torch.empty(
232
scales_and_zp_size,
233
output_size_per_partition,
234
dtype=params_dtype,
235
),
236
- requires_grad=False,
237
- )
238
- set_weight_attrs(
239
- scales,
240
- {
241
- **extra_weight_attrs,
242
- "input_dim": scales_and_zp_input_dim,
243
- "output_dim": 1,
244
- },
245
- )
246
-
247
- # Quantized zero-points
248
- qzeros = Parameter(
249
- torch.empty(
250
- scales_and_zp_size,
251
- output_size_per_partition // self.quant_config.pack_factor,
252
- dtype=torch.int32,
253
- ),
254
- requires_grad=False,
255
- )
256
- set_weight_attrs(
257
- qzeros,
258
- {
259
- **extra_weight_attrs,
260
- "input_dim": scales_and_zp_input_dim,
261
- "output_dim": 1,
262
- "packed_dim": 1,
263
- "pack_factor": self.quant_config.pack_factor,
264
- },
265
- )
236
+ "weight_loader":
237
+ weight_loader
238
+ }
239
+
240
+ if scales_and_zp_input_dim is None:
241
+ scales = ChannelQuantScaleParameter(output_dim=1,
242
+ **weight_scale_args)
243
+ qzeros = PackedColumnParameter(
244
+ output_dim=1,
245
+ packed_dim=1,
246
+ packed_factor=self.quant_config.pack_factor,
247
+ **qzeros_args)
248
+
249
+ else:
250
+ scales = GroupQuantScaleParameter(output_dim=1,
251
+ input_dim=0,
252
+ **weight_scale_args)
253
+ qzeros = PackedvLLMParameter(
254
+ input_dim=0,
255
+ output_dim=1,
256
+ packed_dim=1,
257
+ packed_factor=self.quant_config.pack_factor,
258
+ **qzeros_args)
259
260
layer.register_parameter("qweight", qweight)
261
layer.register_parameter("g_idx", g_idx)
@@ -280,6 +273,10 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
273
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
274
device = layer.qweight.device
275
276
+ # required by torch.compile
277
+ layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
278
+ layer.scales = Parameter(layer.scales.data, requires_grad=False)
279
+
280
# Allocate marlin workspace
281
layer.workspace = marlin_make_workspace(
282
layer.output_size_per_partition, device)
vllm/model_executor/parameter.py
+85
-25
@@ -9,7 +9,7 @@ from vllm.logger import init_logger
9
__all__ = [
10
"BasevLLMParameter", "PackedvLLMParameter", "PerTensorScaleParameter",
11
"ModelWeightParameter", "ChannelQuantScaleParameter",
12
- "GroupQuantScaleParameter"
12
+ "GroupQuantScaleParameter", "PackedColumnParameter", "RowvLLMParameter"
13
]
14
15
logger = init_logger(__name__)
@@ -92,7 +92,8 @@ class _ColumnvLLMParameter(BasevLLMParameter):
92
shard_size = kwargs.get("shard_size")
93
if isinstance(
94
self,
95
- PackedvLLMParameter) and self.packed_dim == self.output_dim:
95
+ (PackedColumnParameter,
96
+ PackedvLLMParameter)) and self.packed_dim == self.output_dim:
97
shard_size, shard_offset = self.adjust_shard_indexes_for_packing(
98
shard_offset=shard_offset, shard_size=shard_size)
99
@@ -115,7 +116,8 @@ class _ColumnvLLMParameter(BasevLLMParameter):
116
117
if isinstance(
118
self,
118
- PackedvLLMParameter) and self.output_dim == self.packed_dim:
119
+ (PackedColumnParameter,
120
+ PackedvLLMParameter)) and self.output_dim == self.packed_dim:
121
shard_size, shard_offset = self.adjust_shard_indexes_for_packing(
122
shard_offset=shard_offset, shard_size=shard_size)
123
@@ -131,12 +133,12 @@ class _ColumnvLLMParameter(BasevLLMParameter):
133
param_data.copy_(loaded_weight)
134
135
134
-class ModelWeightParameter(_ColumnvLLMParameter):
136
+class RowvLLMParameter(BasevLLMParameter):
137
"""
136
- Parameter class for linear layer weights. Extends the
137
- _ColumnvLLMParameter by adding loading functionality
138
- for linear layers with row parallel functionality.
139
- Requires an input dimension to be defined.
138
+ Parameter class defining weight_loading functionality
139
+ (load_row_parallel_weight) for parameters being loaded
140
+ into linear layers with row parallel functionality.
141
+ Requires an input_dim to be defined.
142
"""
143
144
def __init__(self, input_dim: int, **kwargs):
@@ -160,10 +162,18 @@ class ModelWeightParameter(_ColumnvLLMParameter):
162
self.data.copy_(loaded_weight)
163
164
163
-class GroupQuantScaleParameter(ModelWeightParameter):
165
+class ModelWeightParameter(_ColumnvLLMParameter, RowvLLMParameter):
166
+ """
167
+ Parameter class for linear layer weights. Uses both column and
168
+ row parallelism.
169
+ """
170
+ pass
171
+
172
+
173
+class GroupQuantScaleParameter(_ColumnvLLMParameter, RowvLLMParameter):
174
"""
175
Parameter class for weight scales loaded for weights with
166
- grouped quantization. Equivalent to ModelWeightParameter.
176
+ grouped quantization. Uses both column and row parallelism.
177
"""
178
pass
179
@@ -171,7 +181,7 @@ class GroupQuantScaleParameter(ModelWeightParameter):
181
class ChannelQuantScaleParameter(_ColumnvLLMParameter):
182
"""
183
Parameter class for weight scales loaded for weights with
174
- channel-wise quantization. Equivalent to _ColumnvLLMParameter.
184
+ channel-wise quantization. Equivalent to _ColumnvLLMParameter.
185
"""
186
pass
187
@@ -181,7 +191,7 @@ class PerTensorScaleParameter(BasevLLMParameter):
191
Parameter class for scales where the number of scales is
192
equivalent to the number of logical matrices in fused linear
193
layers (e.g. for QKV, there are 3 scales loaded from disk).
184
- This is relevant to weights with per-tensor quantization.
194
+ This is relevant to weights with per-tensor quantization.
195
Adds functionality to map the scalers to a shard during
196
weight loading.
197
@@ -232,6 +242,43 @@ class PerTensorScaleParameter(BasevLLMParameter):
242
param_data.copy_(loaded_weight)
243
244
245
+class PackedColumnParameter(_ColumnvLLMParameter):
246
+ """
247
+ Parameter for model parameters which are packed on disk
248
+ and support column parallelism only. See PackedvLLMParameter
249
+ for more details on the packed properties.
250
+ """
251
+
252
+ def __init__(self,
253
+ packed_factor: int,
254
+ packed_dim: int,
255
+ marlin_tile_size: Optional[int] = None,
256
+ **kwargs):
257
+ self._packed_factor = packed_factor
258
+ self._packed_dim = packed_dim
259
+ self._marlin_tile_size = marlin_tile_size
260
+ super().__init__(**kwargs)
261
+
262
+ @property
263
+ def packed_dim(self):
264
+ return self._packed_dim
265
+
266
+ @property
267
+ def packed_factor(self):
268
+ return self._packed_factor
269
+
270
+ @property
271
+ def marlin_tile_size(self):
272
+ return self._marlin_tile_size
273
+
274
+ def adjust_shard_indexes_for_packing(self, shard_size, shard_offset):
275
+ return _adjust_shard_indexes_for_packing(
276
+ shard_size=shard_size,
277
+ shard_offset=shard_offset,
278
+ packed_factor=self.packed_factor,
279
+ marlin_tile_size=self.marlin_tile_size)
280
+
281
+
282
class PackedvLLMParameter(ModelWeightParameter):
283
"""
284
Parameter for model weights which are packed on disk.
@@ -239,7 +286,7 @@ class PackedvLLMParameter(ModelWeightParameter):
286
Extends the ModelWeightParameter to take in the
287
packed factor, the packed dimension, and optionally, marlin
288
tile size for marlin kernels. Adjusts the shard_size and
242
- shard_offset for fused linear layers model weight loading
289
+ shard_offset for fused linear layers model weight loading
290
by accounting for packing and optionally, marlin tile size.
291
"""
292
@@ -250,7 +297,7 @@ class PackedvLLMParameter(ModelWeightParameter):
297
**kwargs):
298
self._packed_factor = packed_factor
299
self._packed_dim = packed_dim
253
- self._marlin_tile = marlin_tile_size
300
+ self._marlin_tile_size = marlin_tile_size
301
super().__init__(**kwargs)
302
303
@property
@@ -262,16 +309,29 @@ class PackedvLLMParameter(ModelWeightParameter):
309
return self._packed_factor
310
311
@property
265
- def marlin_tile(self):
266
- return self._marlin_tile
267
-
268
- def _adjust_shard_indexes_for_marlin(self, shard_size, shard_offset):
269
- return shard_size * self.marlin_tile, shard_offset * self.marlin_tile
312
+ def marlin_tile_size(self):
313
+ return self._marlin_tile_size
314
315
def adjust_shard_indexes_for_packing(self, shard_size, shard_offset):
272
- shard_size = shard_size // self.packed_factor
273
- shard_offset = shard_offset // self.packed_factor
274
- if self.marlin_tile is not None:
275
- return self._adjust_shard_indexes_for_marlin(
276
- shard_size, shard_offset)
277
- return shard_size, shard_offset
316
+ return _adjust_shard_indexes_for_packing(
317
+ shard_size=shard_size,
318
+ shard_offset=shard_offset,
319
+ packed_factor=self.packed_factor,
320
+ marlin_tile_size=self.marlin_tile_size)
321
+
322
+
323
+def _adjust_shard_indexes_for_marlin(shard_size, shard_offset,
324
+ marlin_tile_size):
325
+ return shard_size * marlin_tile_size, shard_offset * marlin_tile_size
326
+
327
+
328
+def _adjust_shard_indexes_for_packing(shard_size, shard_offset, packed_factor,
329
+ marlin_tile_size):
330
+ shard_size = shard_size // packed_factor
331
+ shard_offset = shard_offset // packed_factor
332
+ if marlin_tile_size is not None:
333
+ return _adjust_shard_indexes_for_marlin(
334
+ shard_size=shard_size,
335
+ shard_offset=shard_offset,
336
+ marlin_tile_size=marlin_tile_size)
337
+ return shard_size, shard_offset