@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b1e5afc3e7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Misc] Update `awq` and `awq_marlin` to use `vLLMParameters` (#7422)

Dipika Sikka committed Aug 13, 2024 at 17:08 UTC b1e5afc3e7843e993d3f7d40e57f0fecb9d137b5
4 files changed +74 -83
tests/weight_loading/models.txt
+3 -1
@@ -12,4 +12,6 @@ compressed-tensors, nm-testing/tinyllama-oneshot-w4a16-group128-v2, main
12 compressed-tensors, nm-testing/tinyllama-oneshot-w8a16-per-channel, main
13 compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main
14 compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
15 -compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
\ No newline at end of file
15 +compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
16 +awq, casperhansen/mixtral-instruct-awq, main
17 +awq_marlin, casperhansen/mixtral-instruct-awq, main
vllm/model_executor/layers/linear.py
+2 -1
@@ -21,7 +21,8 @@ from vllm.model_executor.utils import set_weight_attrs
21 logger = init_logger(__name__)
22
23 WEIGHT_LOADER_V2_SUPPORTED = [
24 - "CompressedTensorsLinearMethod", "GPTQMarlinLinearMethod"
24 + "CompressedTensorsLinearMethod", "AWQMarlinLinearMethod",
25 + "AWQLinearMethod", "GPTQMarlinLinearMethod"
26 ]
27
28
vllm/model_executor/layers/quantization/awq.py
+35 -39
@@ -1,13 +1,13 @@
1 from typing import Any, Dict, List, Optional
2
3 import torch
4 -from torch.nn.parameter import Parameter
4
5 from vllm import _custom_ops as ops
6 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
7 from vllm.model_executor.layers.quantization.base_config import (
8 QuantizationConfig)
10 -from vllm.model_executor.utils import set_weight_attrs
9 +from vllm.model_executor.parameter import (GroupQuantScaleParameter,
10 + PackedvLLMParameter)
11
12
13 class AWQConfig(QuantizationConfig):
@@ -101,55 +101,51 @@ class AWQLinearMethod(LinearMethodBase):
101 "weight shape. This can be caused by too large "
102 "tensor parallel size.")
103
104 - qweight = Parameter(
105 - torch.empty(
104 + weight_loader = extra_weight_attrs.get("weight_loader")
105 + qweight = PackedvLLMParameter(
106 + data=torch.empty(
107 input_size_per_partition,
108 output_size_per_partition // self.quant_config.pack_factor,
109 dtype=torch.int32,
110 ),
110 - requires_grad=False,
111 - )
112 - set_weight_attrs(
113 - qweight, {
114 - "input_dim": 0,
115 - "output_dim": 1,
116 - "packed_dim": 1,
117 - "pack_factor": self.quant_config.pack_factor,
118 - })
119 - qzeros = Parameter(
120 - torch.empty(
111 + input_dim=0,
112 + output_dim=1,
113 + packed_dim=1,
114 + packed_factor=self.quant_config.pack_factor,
115 + weight_loader=weight_loader)
116 +
117 + qzeros = PackedvLLMParameter(
118 + data=torch.empty(
119 input_size_per_partition // self.quant_config.group_size,
120 output_size_per_partition // self.quant_config.pack_factor,
121 dtype=torch.int32,
122 ),
125 - requires_grad=False,
126 - )
127 - set_weight_attrs(
128 - qzeros, {
129 - "input_dim": 0,
130 - "output_dim": 1,
131 - "packed_dim": 1,
132 - "pack_factor": self.quant_config.pack_factor,
133 - })
134 - scales = Parameter(
135 - torch.empty(
136 - input_size_per_partition // self.quant_config.group_size,
137 - output_size_per_partition,
138 - dtype=params_dtype,
139 - ),
140 - requires_grad=False,
141 - )
142 - set_weight_attrs(scales, {
143 - "input_dim": 0,
144 - "output_dim": 1,
145 - })
123 + input_dim=0,
124 + output_dim=1,
125 + packed_dim=1,
126 + packed_factor=self.quant_config.pack_factor,
127 + weight_loader=weight_loader)
128 +
129 + scales = GroupQuantScaleParameter(data=torch.empty(
130 + input_size_per_partition // self.quant_config.group_size,
131 + output_size_per_partition,
132 + dtype=params_dtype,
133 + ),
134 + input_dim=0,
135 + output_dim=1,
136 + weight_loader=weight_loader)
137
138 layer.register_parameter("qweight", qweight)
148 - set_weight_attrs(qweight, extra_weight_attrs)
139 layer.register_parameter("qzeros", qzeros)
150 - set_weight_attrs(qzeros, extra_weight_attrs)
140 layer.register_parameter("scales", scales)
152 - set_weight_attrs(scales, extra_weight_attrs)
141 +
142 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
143 + layer.qweight = torch.nn.Parameter(layer.qweight.data,
144 + requires_grad=False)
145 + layer.qzeros = torch.nn.Parameter(layer.qzeros.data,
146 + requires_grad=False)
147 + layer.scales = torch.nn.Parameter(layer.scales.data,
148 + requires_grad=False)
149
150 def apply(self,
151 layer: torch.nn.Module,
vllm/model_executor/layers/quantization/awq_marlin.py
+34 -42
@@ -1,12 +1,10 @@
1 from typing import Any, Dict, List, Optional
2
3 import torch
4 -from torch.nn.parameter import Parameter
4
5 from vllm import _custom_ops as ops
6 from vllm.logger import init_logger
8 -from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9 - set_weight_attrs)
7 +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8 from vllm.model_executor.layers.quantization.base_config import (
9 QuantizationConfig)
10 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
@@ -14,6 +12,8 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12 marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
13 replace_tensor, verify_marlin_supported, verify_marlin_supports_shape)
14 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
15 +from vllm.model_executor.parameter import (GroupQuantScaleParameter,
16 + PackedvLLMParameter)
17 from vllm.scalar_type import scalar_types
18
19 logger = init_logger(__name__)
@@ -151,6 +151,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
151 ) -> None:
152 del output_size
153 output_size_per_partition = sum(output_partition_sizes)
154 + weight_loader = extra_weight_attrs.get("weight_loader")
155
156 # Normalize group_size
157 if self.quant_config.group_size != -1:
@@ -164,59 +165,44 @@ class AWQMarlinLinearMethod(LinearMethodBase):
165 input_size=input_size,
166 group_size=group_size)
167
167 - qweight = Parameter(
168 - torch.empty(
168 + qweight = PackedvLLMParameter(
169 + data=torch.empty(
170 input_size_per_partition,
171 output_size_per_partition // self.quant_config.pack_factor,
172 dtype=torch.int32,
173 ),
173 - requires_grad=False,
174 - )
175 - set_weight_attrs(
176 - qweight, {
177 - "input_dim": 0,
178 - "output_dim": 1,
179 - "packed_dim": 1,
180 - "pack_factor": self.quant_config.pack_factor,
181 - })
174 + input_dim=0,
175 + output_dim=1,
176 + packed_dim=1,
177 + packed_factor=self.quant_config.pack_factor,
178 + weight_loader=weight_loader)
179
180 num_groups = input_size_per_partition // group_size
181
185 - qzeros = Parameter(
186 - torch.empty(
182 + qzeros = PackedvLLMParameter(
183 + data=torch.empty(
184 num_groups,
185 output_size_per_partition // self.quant_config.pack_factor,
186 dtype=torch.int32,
187 ),
191 - requires_grad=False,
192 - )
193 - set_weight_attrs(
194 - qzeros, {
195 - "input_dim": 0,
196 - "output_dim": 1,
197 - "packed_dim": 1,
198 - "pack_factor": self.quant_config.pack_factor,
199 - })
200 -
201 - scales = Parameter(
202 - torch.empty(
203 - num_groups,
204 - output_size_per_partition,
205 - dtype=params_dtype,
206 - ),
207 - requires_grad=False,
208 - )
209 - set_weight_attrs(scales, {
210 - "input_dim": 0,
211 - "output_dim": 1,
212 - })
188 + input_dim=0,
189 + output_dim=1,
190 + packed_dim=1,
191 + packed_factor=self.quant_config.pack_factor,
192 + weight_loader=weight_loader)
193 +
194 + scales = GroupQuantScaleParameter(data=torch.empty(
195 + num_groups,
196 + output_size_per_partition,
197 + dtype=params_dtype,
198 + ),
199 + input_dim=0,
200 + output_dim=1,
201 + weight_loader=weight_loader)
202
203 layer.register_parameter("qweight", qweight)
215 - set_weight_attrs(qweight, extra_weight_attrs)
204 layer.register_parameter("qzeros", qzeros)
217 - set_weight_attrs(qzeros, extra_weight_attrs)
205 layer.register_parameter("scales", scales)
219 - set_weight_attrs(scales, extra_weight_attrs)
206
207 layer.input_size_per_partition = input_size_per_partition
208 layer.output_size_per_partition = output_size_per_partition
@@ -228,6 +214,12 @@ class AWQMarlinLinearMethod(LinearMethodBase):
214 # Here, we handle the repacking
215 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
216 device = layer.qweight.device
217 + layer.qweight = torch.nn.Parameter(layer.qweight.data,
218 + requires_grad=False)
219 + layer.qzeros = torch.nn.Parameter(layer.qzeros.data,
220 + requires_grad=False)
221 + layer.scales = torch.nn.Parameter(layer.scales.data,
222 + requires_grad=False)
223
224 # Allocate marlin workspace
225 layer.workspace = marlin_make_workspace(
@@ -278,4 +270,4 @@ class AWQMarlinLinearMethod(LinearMethodBase):
270 quant_type=self.quant_config.quant_type,
271 output_size_per_partition=layer.output_size_per_partition,
272 input_size_per_partition=layer.input_size_per_partition,
281 - bias=bias)
273 + bias=bias)
\ No newline at end of file