A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Misc] Update `awq` and `awq_marlin` to use `vLLMParameters` (#7422)
Dipika Sikka committed
Aug 13, 2024 at 17:08 UTC
b1e5afc3e7843e993d3f7d40e57f0fecb9d137b5
4 files changed
+74
-83
tests/weight_loading/models.txt
+3
-1
@@ -12,4 +12,6 @@ compressed-tensors, nm-testing/tinyllama-oneshot-w4a16-group128-v2, main
12
compressed-tensors, nm-testing/tinyllama-oneshot-w8a16-per-channel, main
13
compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main
14
compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
15
-compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
\ No newline at end of file
15
+compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
16
+awq, casperhansen/mixtral-instruct-awq, main
17
+awq_marlin, casperhansen/mixtral-instruct-awq, main
vllm/model_executor/layers/linear.py
+2
-1
@@ -21,7 +21,8 @@ from vllm.model_executor.utils import set_weight_attrs
21
logger = init_logger(__name__)
22
23
WEIGHT_LOADER_V2_SUPPORTED = [
24
- "CompressedTensorsLinearMethod", "GPTQMarlinLinearMethod"
24
+ "CompressedTensorsLinearMethod", "AWQMarlinLinearMethod",
25
+ "AWQLinearMethod", "GPTQMarlinLinearMethod"
26
]
27
28
vllm/model_executor/layers/quantization/awq.py
+35
-39
@@ -1,13 +1,13 @@
1
from typing import Any, Dict, List, Optional
2
3
import torch
4
-from torch.nn.parameter import Parameter
4
5
from vllm import _custom_ops as ops
6
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
7
from vllm.model_executor.layers.quantization.base_config import (
8
QuantizationConfig)
10
-from vllm.model_executor.utils import set_weight_attrs
9
+from vllm.model_executor.parameter import (GroupQuantScaleParameter,
10
+ PackedvLLMParameter)
11
12
13
class AWQConfig(QuantizationConfig):
@@ -101,55 +101,51 @@ class AWQLinearMethod(LinearMethodBase):
101
"weight shape. This can be caused by too large "
102
"tensor parallel size.")
103
104
- qweight = Parameter(
105
- torch.empty(
104
+ weight_loader = extra_weight_attrs.get("weight_loader")
105
+ qweight = PackedvLLMParameter(
106
+ data=torch.empty(
107
input_size_per_partition,
108
output_size_per_partition // self.quant_config.pack_factor,
109
dtype=torch.int32,
110
),
110
- requires_grad=False,
111
- )
112
- set_weight_attrs(
113
- qweight, {
114
- "input_dim": 0,
115
- "output_dim": 1,
116
- "packed_dim": 1,
117
- "pack_factor": self.quant_config.pack_factor,
118
- })
119
- qzeros = Parameter(
120
- torch.empty(
111
+ input_dim=0,
112
+ output_dim=1,
113
+ packed_dim=1,
114
+ packed_factor=self.quant_config.pack_factor,
115
+ weight_loader=weight_loader)
116
+
117
+ qzeros = PackedvLLMParameter(
118
+ data=torch.empty(
119
input_size_per_partition // self.quant_config.group_size,
120
output_size_per_partition // self.quant_config.pack_factor,
121
dtype=torch.int32,
122
),
125
- requires_grad=False,
126
- )
127
- set_weight_attrs(
128
- qzeros, {
129
- "input_dim": 0,
130
- "output_dim": 1,
131
- "packed_dim": 1,
132
- "pack_factor": self.quant_config.pack_factor,
133
- })
134
- scales = Parameter(
135
- torch.empty(
136
- input_size_per_partition // self.quant_config.group_size,
137
- output_size_per_partition,
138
- dtype=params_dtype,
139
- ),
140
- requires_grad=False,
141
- )
142
- set_weight_attrs(scales, {
143
- "input_dim": 0,
144
- "output_dim": 1,
145
- })
123
+ input_dim=0,
124
+ output_dim=1,
125
+ packed_dim=1,
126
+ packed_factor=self.quant_config.pack_factor,
127
+ weight_loader=weight_loader)
128
+
129
+ scales = GroupQuantScaleParameter(data=torch.empty(
130
+ input_size_per_partition // self.quant_config.group_size,
131
+ output_size_per_partition,
132
+ dtype=params_dtype,
133
+ ),
134
+ input_dim=0,
135
+ output_dim=1,
136
+ weight_loader=weight_loader)
137
138
layer.register_parameter("qweight", qweight)
148
- set_weight_attrs(qweight, extra_weight_attrs)
139
layer.register_parameter("qzeros", qzeros)
150
- set_weight_attrs(qzeros, extra_weight_attrs)
140
layer.register_parameter("scales", scales)
152
- set_weight_attrs(scales, extra_weight_attrs)
141
+
142
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
143
+ layer.qweight = torch.nn.Parameter(layer.qweight.data,
144
+ requires_grad=False)
145
+ layer.qzeros = torch.nn.Parameter(layer.qzeros.data,
146
+ requires_grad=False)
147
+ layer.scales = torch.nn.Parameter(layer.scales.data,
148
+ requires_grad=False)
149
150
def apply(self,
151
layer: torch.nn.Module,
vllm/model_executor/layers/quantization/awq_marlin.py
+34
-42
@@ -1,12 +1,10 @@
1
from typing import Any, Dict, List, Optional
2
3
import torch
4
-from torch.nn.parameter import Parameter
4
5
from vllm import _custom_ops as ops
6
from vllm.logger import init_logger
8
-from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9
- set_weight_attrs)
7
+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8
from vllm.model_executor.layers.quantization.base_config import (
9
QuantizationConfig)
10
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
@@ -14,6 +12,8 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12
marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
13
replace_tensor, verify_marlin_supported, verify_marlin_supports_shape)
14
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
15
+from vllm.model_executor.parameter import (GroupQuantScaleParameter,
16
+ PackedvLLMParameter)
17
from vllm.scalar_type import scalar_types
18
19
logger = init_logger(__name__)
@@ -151,6 +151,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
151
) -> None:
152
del output_size
153
output_size_per_partition = sum(output_partition_sizes)
154
+ weight_loader = extra_weight_attrs.get("weight_loader")
155
156
# Normalize group_size
157
if self.quant_config.group_size != -1:
@@ -164,59 +165,44 @@ class AWQMarlinLinearMethod(LinearMethodBase):
165
input_size=input_size,
166
group_size=group_size)
167
167
- qweight = Parameter(
168
- torch.empty(
168
+ qweight = PackedvLLMParameter(
169
+ data=torch.empty(
170
input_size_per_partition,
171
output_size_per_partition // self.quant_config.pack_factor,
172
dtype=torch.int32,
173
),
173
- requires_grad=False,
174
- )
175
- set_weight_attrs(
176
- qweight, {
177
- "input_dim": 0,
178
- "output_dim": 1,
179
- "packed_dim": 1,
180
- "pack_factor": self.quant_config.pack_factor,
181
- })
174
+ input_dim=0,
175
+ output_dim=1,
176
+ packed_dim=1,
177
+ packed_factor=self.quant_config.pack_factor,
178
+ weight_loader=weight_loader)
179
180
num_groups = input_size_per_partition // group_size
181
185
- qzeros = Parameter(
186
- torch.empty(
182
+ qzeros = PackedvLLMParameter(
183
+ data=torch.empty(
184
num_groups,
185
output_size_per_partition // self.quant_config.pack_factor,
186
dtype=torch.int32,
187
),
191
- requires_grad=False,
192
- )
193
- set_weight_attrs(
194
- qzeros, {
195
- "input_dim": 0,
196
- "output_dim": 1,
197
- "packed_dim": 1,
198
- "pack_factor": self.quant_config.pack_factor,
199
- })
200
-
201
- scales = Parameter(
202
- torch.empty(
203
- num_groups,
204
- output_size_per_partition,
205
- dtype=params_dtype,
206
- ),
207
- requires_grad=False,
208
- )
209
- set_weight_attrs(scales, {
210
- "input_dim": 0,
211
- "output_dim": 1,
212
- })
188
+ input_dim=0,
189
+ output_dim=1,
190
+ packed_dim=1,
191
+ packed_factor=self.quant_config.pack_factor,
192
+ weight_loader=weight_loader)
193
+
194
+ scales = GroupQuantScaleParameter(data=torch.empty(
195
+ num_groups,
196
+ output_size_per_partition,
197
+ dtype=params_dtype,
198
+ ),
199
+ input_dim=0,
200
+ output_dim=1,
201
+ weight_loader=weight_loader)
202
203
layer.register_parameter("qweight", qweight)
215
- set_weight_attrs(qweight, extra_weight_attrs)
204
layer.register_parameter("qzeros", qzeros)
217
- set_weight_attrs(qzeros, extra_weight_attrs)
205
layer.register_parameter("scales", scales)
219
- set_weight_attrs(scales, extra_weight_attrs)
206
207
layer.input_size_per_partition = input_size_per_partition
208
layer.output_size_per_partition = output_size_per_partition
@@ -228,6 +214,12 @@ class AWQMarlinLinearMethod(LinearMethodBase):
214
# Here, we handle the repacking
215
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
216
device = layer.qweight.device
217
+ layer.qweight = torch.nn.Parameter(layer.qweight.data,
218
+ requires_grad=False)
219
+ layer.qzeros = torch.nn.Parameter(layer.qzeros.data,
220
+ requires_grad=False)
221
+ layer.scales = torch.nn.Parameter(layer.scales.data,
222
+ requires_grad=False)
223
224
# Allocate marlin workspace
225
layer.workspace = marlin_make_workspace(
@@ -278,4 +270,4 @@ class AWQMarlinLinearMethod(LinearMethodBase):
270
quant_type=self.quant_config.quant_type,
271
output_size_per_partition=layer.output_size_per_partition,
272
input_size_per_partition=layer.input_size_per_partition,
281
- bias=bias)
273
+ bias=bias)
\ No newline at end of file