@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2188a60c7e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Misc] Update `GPTQ` to use `vLLMParameters` (#7976)

Dipika Sikka committed Sep 3, 2024 at 17:21 UTC 2188a60c7e0e5a414a87a4f0fd798333b2e0f625
6 files changed +93 -62
tests/weight_loading/models.txt
+6
@@ -4,6 +4,12 @@ gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, main
4 gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit--1g-actorder_True
5 gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit-32g-actorder_True
6 gptq_marlin, TechxGenus/gemma-1.1-2b-it-GPTQ, main
7 +gptq, robertgshaw2/zephyr-7b-beta-channelwise-gptq, main
8 +gptq, TheBloke/Llama-2-7B-GPTQ, main
9 +gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, main
10 +gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit--1g-actorder_True
11 +gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit-32g-actorder_True
12 +gptq, TechxGenus/gemma-1.1-2b-it-GPTQ, main
13 compressed-tensors, nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change, main
14 compressed-tensors, nm-testing/tinyllama-oneshot-w8-channel-a8-tensor, main
15 compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2, main
tests/weight_loading/test_weight_loading.py
+6 -1
@@ -1,5 +1,7 @@
1 import os
2
3 +import torch
4 +
5 MAX_MODEL_LEN = 1024
6 MODEL_NAME = os.environ.get("MODEL_NAME",
7 "robertgshaw2/zephyr-7b-beta-channelwise-gptq")
@@ -8,9 +10,12 @@ QUANTIZATION = os.environ.get("QUANTIZATION", "gptq_marlin")
10
11
12 def test_weight_loading(vllm_runner):
13 + """
14 + Test parameter weight loading with tp>1.
15 + """
16 with vllm_runner(model_name=MODEL_NAME,
17 revision=REVISION,
13 - dtype="auto",
18 + dtype=torch.half if QUANTIZATION == "gptq" else "auto",
19 quantization=QUANTIZATION,
20 max_model_len=MAX_MODEL_LEN,
21 tensor_parallel_size=2) as model:
vllm/model_executor/layers/linear.py
+14 -11
@@ -14,8 +14,10 @@ from vllm.logger import init_logger
14 from vllm.model_executor.layers.quantization.base_config import (
15 QuantizationConfig, QuantizeMethodBase)
16 from vllm.model_executor.parameter import (BasevLLMParameter,
17 + PackedColumnParameter,
18 PackedvLLMParameter,
18 - PerTensorScaleParameter)
19 + PerTensorScaleParameter,
20 + RowvLLMParameter)
21 from vllm.model_executor.utils import set_weight_attrs
22
23 logger = init_logger(__name__)
@@ -24,7 +26,7 @@ WEIGHT_LOADER_V2_SUPPORTED = [
26 "CompressedTensorsLinearMethod", "AWQMarlinLinearMethod",
27 "AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod",
28 "MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod",
27 - "TPUInt8LinearMethod"
29 + "TPUInt8LinearMethod", "GPTQLinearMethod"
30 ]
31
32
@@ -574,8 +576,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
576 # Special case for Quantization.
577 # If quantized, we need to adjust the offset and size to account
578 # for the packing.
577 - if isinstance(param, PackedvLLMParameter
578 - ) and param.packed_dim == param.output_dim:
579 + if isinstance(param, (PackedColumnParameter, PackedvLLMParameter
580 + )) and param.packed_dim == param.output_dim:
581 shard_size, shard_offset = \
582 param.adjust_shard_indexes_for_packing(
583 shard_size=shard_size, shard_offset=shard_offset)
@@ -594,9 +596,10 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
596 param.load_merged_column_weight(loaded_weight=loaded_weight,
597 shard_id=0)
598 return
597 - elif type(param) is BasevLLMParameter:
599 + elif type(param) in (RowvLLMParameter, BasevLLMParameter):
600 param.load_merged_column_weight(loaded_weight=loaded_weight)
601 return
602 + # TODO: @dsikka - move to parameter.py
603 self._load_fused_module_from_checkpoint(param, loaded_weight)
604 return
605
@@ -724,8 +727,8 @@ class QKVParallelLinear(ColumnParallelLinear):
727 # Special case for Quantization.
728 # If quantized, we need to adjust the offset and size to account
729 # for the packing.
727 - if isinstance(param, PackedvLLMParameter
728 - ) and param.packed_dim == param.output_dim:
730 + if isinstance(param, (PackedColumnParameter, PackedvLLMParameter
731 + )) and param.packed_dim == param.output_dim:
732 shard_size, shard_offset = \
733 param.adjust_shard_indexes_for_packing(
734 shard_size=shard_size, shard_offset=shard_offset)
@@ -741,12 +744,12 @@ class QKVParallelLinear(ColumnParallelLinear):
744 loaded_shard_id: Optional[str] = None):
745 if loaded_shard_id is None: # special case for certain models
746 if isinstance(param, PerTensorScaleParameter):
744 - param.load_merged_column_weight(loaded_weight=loaded_weight,
745 - shard_id=0)
747 + param.load_qkv_weight(loaded_weight=loaded_weight, shard_id=0)
748 return
747 - elif type(param) is BasevLLMParameter:
748 - param.load_merged_column_weight(loaded_weight=loaded_weight)
749 + elif type(param) in (RowvLLMParameter, BasevLLMParameter):
750 + param.load_qkv_weight(loaded_weight=loaded_weight)
751 return
752 + # TODO: @dsikka - move to parameter.py
753 self._load_fused_module_from_checkpoint(param, loaded_weight)
754 return
755
vllm/model_executor/layers/quantization/gptq.py
+58 -45
@@ -11,7 +11,11 @@ from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
11 from vllm.model_executor.layers.quantization.base_config import (
12 QuantizationConfig)
13 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
14 -from vllm.model_executor.utils import set_weight_attrs
14 +from vllm.model_executor.parameter import (ChannelQuantScaleParameter,
15 + GroupQuantScaleParameter,
16 + PackedColumnParameter,
17 + PackedvLLMParameter,
18 + RowvLLMParameter)
19
20
21 class GPTQConfig(QuantizationConfig):
@@ -108,6 +112,7 @@ class GPTQLinearMethod(LinearMethodBase):
112 **extra_weight_attrs,
113 ):
114 del output_size # Unused.
115 + weight_loader = extra_weight_attrs.get("weight_loader")
116 if input_size_per_partition % self.quant_config.group_size != 0:
117 raise ValueError(
118 "The input size is not aligned with the quantized "
@@ -138,73 +143,81 @@ class GPTQLinearMethod(LinearMethodBase):
143 scale_and_zero_size = input_size_per_partition // group_size
144 scale_and_zero_input_dim = 0
145
141 - qweight = Parameter(
142 - torch.empty(
146 + qweight = PackedvLLMParameter(
147 + data=torch.empty(
148 input_size_per_partition // self.quant_config.pack_factor,
149 output_size_per_partition,
150 dtype=torch.int32,
151 ),
147 - requires_grad=False,
148 - )
149 - set_weight_attrs(
150 - qweight, {
151 - "input_dim": 0,
152 - "output_dim": 1,
153 - "packed_dim": 0,
154 - "pack_factor": self.quant_config.pack_factor,
155 - })
156 - g_idx = Parameter(
157 - torch.tensor(
158 - [
159 - i // self.quant_config.group_size
160 - for i in range(input_size_per_partition)
161 - ],
162 - dtype=torch.int32,
163 - ),
164 - requires_grad=False,
165 - )
166 - # Ignore warning from fused linear layers such as QKVParallelLinear.
167 - set_weight_attrs(g_idx, {"input_dim": 0, "ignore_warning": True})
168 - qzeros = Parameter(
152 + input_dim=0,
153 + output_dim=1,
154 + packed_dim=0,
155 + packed_factor=self.quant_config.pack_factor,
156 + weight_loader=weight_loader)
157 +
158 + g_idx = RowvLLMParameter(data=torch.tensor(
159 + [
160 + i // self.quant_config.group_size
161 + for i in range(input_size_per_partition)
162 + ],
163 + dtype=torch.int32,
164 + ),
165 + input_dim=0,
166 + weight_loader=weight_loader)
167 + qzeros_args = {
168 + "data":
169 torch.empty(
170 scale_and_zero_size,
171 output_size_per_partition // self.quant_config.pack_factor,
172 dtype=torch.int32,
173 ),
174 - requires_grad=False,
175 - )
176 - set_weight_attrs(
177 - qzeros, {
178 - "input_dim": scale_and_zero_input_dim,
179 - "output_dim": 1,
180 - "packed_dim": 1,
181 - "pack_factor": self.quant_config.pack_factor,
182 - })
183 - scales = Parameter(
174 + "weight_loader":
175 + weight_loader
176 + }
177 + weight_scale_args = {
178 + "data":
179 torch.empty(
180 scale_and_zero_size,
181 output_size_per_partition,
182 dtype=params_dtype,
183 ),
189 - requires_grad=False,
190 - )
191 - set_weight_attrs(scales, {
192 - "input_dim": scale_and_zero_input_dim,
193 - "output_dim": 1,
194 - })
184 + "weight_loader":
185 + weight_loader
186 + }
187 + if scale_and_zero_input_dim is None:
188 + scales = ChannelQuantScaleParameter(output_dim=1,
189 + **weight_scale_args)
190 + qzeros = PackedColumnParameter(
191 + output_dim=1,
192 + packed_dim=1,
193 + packed_factor=self.quant_config.pack_factor,
194 + **qzeros_args)
195 +
196 + else:
197 + scales = GroupQuantScaleParameter(output_dim=1,
198 + input_dim=0,
199 + **weight_scale_args)
200 + qzeros = PackedvLLMParameter(
201 + input_dim=0,
202 + output_dim=1,
203 + packed_dim=1,
204 + packed_factor=self.quant_config.pack_factor,
205 + **qzeros_args)
206
207 layer.register_parameter("qweight", qweight)
197 - set_weight_attrs(qweight, extra_weight_attrs)
208 layer.register_parameter("g_idx", g_idx)
199 - set_weight_attrs(g_idx, extra_weight_attrs)
209 layer.register_parameter("qzeros", qzeros)
201 - set_weight_attrs(qzeros, extra_weight_attrs)
210 layer.register_parameter("scales", scales)
203 - set_weight_attrs(scales, extra_weight_attrs)
211
212 layer.exllama_state = exllama_state
213
214 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
215 + # for torch.compile
216 + layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
217 + layer.qzeros = Parameter(layer.qzeros.data, requires_grad=False)
218 + layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
219 + layer.g_idx = Parameter(layer.g_idx.data, requires_grad=False)
220 +
221 # exllama needs to shuffle the weight after the weight is loaded
222 # here we do the shuffle on first forward pass
223 if layer.exllama_state == ExllamaState.UNINITIALIZED:
vllm/model_executor/layers/vocab_parallel_embedding.py
+6 -3
@@ -10,6 +10,7 @@ from vllm.distributed import (divide, get_tensor_model_parallel_rank,
10 tensor_model_parallel_all_reduce)
11 from vllm.model_executor.layers.quantization.base_config import (
12 QuantizationConfig, QuantizeMethodBase, method_has_implemented_embedding)
13 +from vllm.model_executor.parameter import BasevLLMParameter
14 from vllm.model_executor.utils import set_weight_attrs
15
16 DEFAULT_VOCAB_PADDING_SIZE = 64
@@ -370,10 +371,12 @@ class VocabParallelEmbedding(torch.nn.Module):
371 # If param packed on the same dim we are sharding on, then
372 # need to adjust offsets of loaded weight by pack_factor.
373 if packed_dim is not None and packed_dim == output_dim:
374 + packed_factor = param.packed_factor if isinstance(
375 + param, BasevLLMParameter) else param.pack_factor
376 assert loaded_weight.shape[output_dim] == (self.org_vocab_size //
374 - param.pack_factor)
375 - start_idx = start_idx // param.pack_factor
376 - shard_size = shard_size // param.pack_factor
377 + param.packed_factor)
378 + start_idx = start_idx // packed_factor
379 + shard_size = shard_size // packed_factor
380 else:
381 assert loaded_weight.shape[output_dim] == self.org_vocab_size
382
vllm/model_executor/parameter.py
+3 -2
@@ -1,3 +1,4 @@
1 +from fractions import Fraction
2 from typing import Callable, Optional, Union
3
4 import torch
@@ -257,7 +258,7 @@ class PackedColumnParameter(_ColumnvLLMParameter):
258 """
259
260 def __init__(self,
260 - packed_factor: int,
261 + packed_factor: Union[int, Fraction],
262 packed_dim: int,
263 marlin_tile_size: Optional[int] = None,
264 **kwargs):
@@ -298,7 +299,7 @@ class PackedvLLMParameter(ModelWeightParameter):
299 """
300
301 def __init__(self,
301 - packed_factor: int,
302 + packed_factor: Union[int, Fraction],
303 packed_dim: int,
304 marlin_tile_size: Optional[int] = None,
305 **kwargs):