A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Misc] Update `GPTQ` to use `vLLMParameters` (#7976)
Dipika Sikka committed
Sep 3, 2024 at 17:21 UTC
2188a60c7e0e5a414a87a4f0fd798333b2e0f625
6 files changed
+93
-62
tests/weight_loading/models.txt
+6
@@ -4,6 +4,12 @@ gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, main
4
gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit--1g-actorder_True
5
gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit-32g-actorder_True
6
gptq_marlin, TechxGenus/gemma-1.1-2b-it-GPTQ, main
7
+gptq, robertgshaw2/zephyr-7b-beta-channelwise-gptq, main
8
+gptq, TheBloke/Llama-2-7B-GPTQ, main
9
+gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, main
10
+gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit--1g-actorder_True
11
+gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit-32g-actorder_True
12
+gptq, TechxGenus/gemma-1.1-2b-it-GPTQ, main
13
compressed-tensors, nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change, main
14
compressed-tensors, nm-testing/tinyllama-oneshot-w8-channel-a8-tensor, main
15
compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2, main
tests/weight_loading/test_weight_loading.py
+6
-1
@@ -1,5 +1,7 @@
1
import os
2
3
+import torch
4
+
5
MAX_MODEL_LEN = 1024
6
MODEL_NAME = os.environ.get("MODEL_NAME",
7
"robertgshaw2/zephyr-7b-beta-channelwise-gptq")
@@ -8,9 +10,12 @@ QUANTIZATION = os.environ.get("QUANTIZATION", "gptq_marlin")
10
11
12
def test_weight_loading(vllm_runner):
13
+ """
14
+ Test parameter weight loading with tp>1.
15
+ """
16
with vllm_runner(model_name=MODEL_NAME,
17
revision=REVISION,
13
- dtype="auto",
18
+ dtype=torch.half if QUANTIZATION == "gptq" else "auto",
19
quantization=QUANTIZATION,
20
max_model_len=MAX_MODEL_LEN,
21
tensor_parallel_size=2) as model:
vllm/model_executor/layers/linear.py
+14
-11
@@ -14,8 +14,10 @@ from vllm.logger import init_logger
14
from vllm.model_executor.layers.quantization.base_config import (
15
QuantizationConfig, QuantizeMethodBase)
16
from vllm.model_executor.parameter import (BasevLLMParameter,
17
+ PackedColumnParameter,
18
PackedvLLMParameter,
18
- PerTensorScaleParameter)
19
+ PerTensorScaleParameter,
20
+ RowvLLMParameter)
21
from vllm.model_executor.utils import set_weight_attrs
22
23
logger = init_logger(__name__)
@@ -24,7 +26,7 @@ WEIGHT_LOADER_V2_SUPPORTED = [
26
"CompressedTensorsLinearMethod", "AWQMarlinLinearMethod",
27
"AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod",
28
"MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod",
27
- "TPUInt8LinearMethod"
29
+ "TPUInt8LinearMethod", "GPTQLinearMethod"
30
]
31
32
@@ -574,8 +576,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
576
# Special case for Quantization.
577
# If quantized, we need to adjust the offset and size to account
578
# for the packing.
577
- if isinstance(param, PackedvLLMParameter
578
- ) and param.packed_dim == param.output_dim:
579
+ if isinstance(param, (PackedColumnParameter, PackedvLLMParameter
580
+ )) and param.packed_dim == param.output_dim:
581
shard_size, shard_offset = \
582
param.adjust_shard_indexes_for_packing(
583
shard_size=shard_size, shard_offset=shard_offset)
@@ -594,9 +596,10 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
596
param.load_merged_column_weight(loaded_weight=loaded_weight,
597
shard_id=0)
598
return
597
- elif type(param) is BasevLLMParameter:
599
+ elif type(param) in (RowvLLMParameter, BasevLLMParameter):
600
param.load_merged_column_weight(loaded_weight=loaded_weight)
601
return
602
+ # TODO: @dsikka - move to parameter.py
603
self._load_fused_module_from_checkpoint(param, loaded_weight)
604
return
605
@@ -724,8 +727,8 @@ class QKVParallelLinear(ColumnParallelLinear):
727
# Special case for Quantization.
728
# If quantized, we need to adjust the offset and size to account
729
# for the packing.
727
- if isinstance(param, PackedvLLMParameter
728
- ) and param.packed_dim == param.output_dim:
730
+ if isinstance(param, (PackedColumnParameter, PackedvLLMParameter
731
+ )) and param.packed_dim == param.output_dim:
732
shard_size, shard_offset = \
733
param.adjust_shard_indexes_for_packing(
734
shard_size=shard_size, shard_offset=shard_offset)
@@ -741,12 +744,12 @@ class QKVParallelLinear(ColumnParallelLinear):
744
loaded_shard_id: Optional[str] = None):
745
if loaded_shard_id is None: # special case for certain models
746
if isinstance(param, PerTensorScaleParameter):
744
- param.load_merged_column_weight(loaded_weight=loaded_weight,
745
- shard_id=0)
747
+ param.load_qkv_weight(loaded_weight=loaded_weight, shard_id=0)
748
return
747
- elif type(param) is BasevLLMParameter:
748
- param.load_merged_column_weight(loaded_weight=loaded_weight)
749
+ elif type(param) in (RowvLLMParameter, BasevLLMParameter):
750
+ param.load_qkv_weight(loaded_weight=loaded_weight)
751
return
752
+ # TODO: @dsikka - move to parameter.py
753
self._load_fused_module_from_checkpoint(param, loaded_weight)
754
return
755
vllm/model_executor/layers/quantization/gptq.py
+58
-45
@@ -11,7 +11,11 @@ from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
11
from vllm.model_executor.layers.quantization.base_config import (
12
QuantizationConfig)
13
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
14
-from vllm.model_executor.utils import set_weight_attrs
14
+from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
15
+ GroupQuantScaleParameter,
16
+ PackedColumnParameter,
17
+ PackedvLLMParameter,
18
+ RowvLLMParameter)
19
20
21
class GPTQConfig(QuantizationConfig):
@@ -108,6 +112,7 @@ class GPTQLinearMethod(LinearMethodBase):
112
**extra_weight_attrs,
113
):
114
del output_size # Unused.
115
+ weight_loader = extra_weight_attrs.get("weight_loader")
116
if input_size_per_partition % self.quant_config.group_size != 0:
117
raise ValueError(
118
"The input size is not aligned with the quantized "
@@ -138,73 +143,81 @@ class GPTQLinearMethod(LinearMethodBase):
143
scale_and_zero_size = input_size_per_partition // group_size
144
scale_and_zero_input_dim = 0
145
141
- qweight = Parameter(
142
- torch.empty(
146
+ qweight = PackedvLLMParameter(
147
+ data=torch.empty(
148
input_size_per_partition // self.quant_config.pack_factor,
149
output_size_per_partition,
150
dtype=torch.int32,
151
),
147
- requires_grad=False,
148
- )
149
- set_weight_attrs(
150
- qweight, {
151
- "input_dim": 0,
152
- "output_dim": 1,
153
- "packed_dim": 0,
154
- "pack_factor": self.quant_config.pack_factor,
155
- })
156
- g_idx = Parameter(
157
- torch.tensor(
158
- [
159
- i // self.quant_config.group_size
160
- for i in range(input_size_per_partition)
161
- ],
162
- dtype=torch.int32,
163
- ),
164
- requires_grad=False,
165
- )
166
- # Ignore warning from fused linear layers such as QKVParallelLinear.
167
- set_weight_attrs(g_idx, {"input_dim": 0, "ignore_warning": True})
168
- qzeros = Parameter(
152
+ input_dim=0,
153
+ output_dim=1,
154
+ packed_dim=0,
155
+ packed_factor=self.quant_config.pack_factor,
156
+ weight_loader=weight_loader)
157
+
158
+ g_idx = RowvLLMParameter(data=torch.tensor(
159
+ [
160
+ i // self.quant_config.group_size
161
+ for i in range(input_size_per_partition)
162
+ ],
163
+ dtype=torch.int32,
164
+ ),
165
+ input_dim=0,
166
+ weight_loader=weight_loader)
167
+ qzeros_args = {
168
+ "data":
169
torch.empty(
170
scale_and_zero_size,
171
output_size_per_partition // self.quant_config.pack_factor,
172
dtype=torch.int32,
173
),
174
- requires_grad=False,
175
- )
176
- set_weight_attrs(
177
- qzeros, {
178
- "input_dim": scale_and_zero_input_dim,
179
- "output_dim": 1,
180
- "packed_dim": 1,
181
- "pack_factor": self.quant_config.pack_factor,
182
- })
183
- scales = Parameter(
174
+ "weight_loader":
175
+ weight_loader
176
+ }
177
+ weight_scale_args = {
178
+ "data":
179
torch.empty(
180
scale_and_zero_size,
181
output_size_per_partition,
182
dtype=params_dtype,
183
),
189
- requires_grad=False,
190
- )
191
- set_weight_attrs(scales, {
192
- "input_dim": scale_and_zero_input_dim,
193
- "output_dim": 1,
194
- })
184
+ "weight_loader":
185
+ weight_loader
186
+ }
187
+ if scale_and_zero_input_dim is None:
188
+ scales = ChannelQuantScaleParameter(output_dim=1,
189
+ **weight_scale_args)
190
+ qzeros = PackedColumnParameter(
191
+ output_dim=1,
192
+ packed_dim=1,
193
+ packed_factor=self.quant_config.pack_factor,
194
+ **qzeros_args)
195
+
196
+ else:
197
+ scales = GroupQuantScaleParameter(output_dim=1,
198
+ input_dim=0,
199
+ **weight_scale_args)
200
+ qzeros = PackedvLLMParameter(
201
+ input_dim=0,
202
+ output_dim=1,
203
+ packed_dim=1,
204
+ packed_factor=self.quant_config.pack_factor,
205
+ **qzeros_args)
206
207
layer.register_parameter("qweight", qweight)
197
- set_weight_attrs(qweight, extra_weight_attrs)
208
layer.register_parameter("g_idx", g_idx)
199
- set_weight_attrs(g_idx, extra_weight_attrs)
209
layer.register_parameter("qzeros", qzeros)
201
- set_weight_attrs(qzeros, extra_weight_attrs)
210
layer.register_parameter("scales", scales)
203
- set_weight_attrs(scales, extra_weight_attrs)
211
212
layer.exllama_state = exllama_state
213
214
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
215
+ # for torch.compile
216
+ layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
217
+ layer.qzeros = Parameter(layer.qzeros.data, requires_grad=False)
218
+ layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
219
+ layer.g_idx = Parameter(layer.g_idx.data, requires_grad=False)
220
+
221
# exllama needs to shuffle the weight after the weight is loaded
222
# here we do the shuffle on first forward pass
223
if layer.exllama_state == ExllamaState.UNINITIALIZED:
vllm/model_executor/layers/vocab_parallel_embedding.py
+6
-3
@@ -10,6 +10,7 @@ from vllm.distributed import (divide, get_tensor_model_parallel_rank,
10
tensor_model_parallel_all_reduce)
11
from vllm.model_executor.layers.quantization.base_config import (
12
QuantizationConfig, QuantizeMethodBase, method_has_implemented_embedding)
13
+from vllm.model_executor.parameter import BasevLLMParameter
14
from vllm.model_executor.utils import set_weight_attrs
15
16
DEFAULT_VOCAB_PADDING_SIZE = 64
@@ -370,10 +371,12 @@ class VocabParallelEmbedding(torch.nn.Module):
371
# If param packed on the same dim we are sharding on, then
372
# need to adjust offsets of loaded weight by pack_factor.
373
if packed_dim is not None and packed_dim == output_dim:
374
+ packed_factor = param.packed_factor if isinstance(
375
+ param, BasevLLMParameter) else param.pack_factor
376
assert loaded_weight.shape[output_dim] == (self.org_vocab_size //
374
- param.pack_factor)
375
- start_idx = start_idx // param.pack_factor
376
- shard_size = shard_size // param.pack_factor
377
+ param.packed_factor)
378
+ start_idx = start_idx // packed_factor
379
+ shard_size = shard_size // packed_factor
380
else:
381
assert loaded_weight.shape[output_dim] == self.org_vocab_size
382
vllm/model_executor/parameter.py
+3
-2
@@ -1,3 +1,4 @@
1
+from fractions import Fraction
2
from typing import Callable, Optional, Union
3
4
import torch
@@ -257,7 +258,7 @@ class PackedColumnParameter(_ColumnvLLMParameter):
258
"""
259
260
def __init__(self,
260
- packed_factor: int,
261
+ packed_factor: Union[int, Fraction],
262
packed_dim: int,
263
marlin_tile_size: Optional[int] = None,
264
**kwargs):
@@ -298,7 +299,7 @@ class PackedvLLMParameter(ModelWeightParameter):
299
"""
300
301
def __init__(self,
301
- packed_factor: int,
302
+ packed_factor: Union[int, Fraction],
303
packed_dim: int,
304
marlin_tile_size: Optional[int] = None,
305
**kwargs):