A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Misc] GPTQ Activation Ordering (#8135)
Kyle Sayers committed
Sep 9, 2024 at 16:27 UTC
c7cb5c333564cb00fc4f6a99d32c35e9ebc0f1ed
4 files changed
+64
-15
tests/weight_loading/models.txt
+1
@@ -21,6 +21,7 @@ compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
21
compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
22
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
23
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
24
+compressed-tensors, nm-testing/TinyLlama-1.1B-Chat-v1.0-actorder-group, main
25
awq, casperhansen/mixtral-instruct-awq, main
26
awq_marlin, casperhansen/mixtral-instruct-awq, main
27
fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+2
-1
@@ -232,7 +232,8 @@ class CompressedTensorsConfig(QuantizationConfig):
232
return CompressedTensorsWNA16(
233
num_bits=weight_quant.num_bits,
234
strategy=weight_quant.strategy,
235
- group_size=weight_quant.group_size)
235
+ group_size=weight_quant.group_size,
236
+ actorder=weight_quant.actorder)
237
238
# Detect If Activation Quantization.
239
# TODO @dsikka: clean-up conditions
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+33
-12
@@ -5,14 +5,18 @@ import torch
5
from vllm import _custom_ops as ops
6
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7
CompressedTensorsScheme)
8
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9
+ ActivationOrdering)
10
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
11
apply_gptq_marlin_linear, marlin_make_empty_g_idx, marlin_make_workspace,
10
- marlin_permute_scales, replace_tensor, verify_marlin_supported,
12
+ marlin_permute_scales, marlin_repeat_scales_on_all_ranks,
13
+ marlin_sort_g_idx, replace_tensor, verify_marlin_supported,
14
verify_marlin_supports_shape)
15
from vllm.model_executor.parameter import (BasevLLMParameter,
16
ChannelQuantScaleParameter,
17
GroupQuantScaleParameter,
15
- PackedvLLMParameter)
18
+ PackedvLLMParameter,
19
+ RowvLLMParameter)
20
from vllm.scalar_type import scalar_types
21
22
__all__ = ["CompressedTensorsWNA16"]
@@ -28,11 +32,13 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
32
def __init__(self,
33
strategy: str,
34
num_bits: int,
31
- group_size: Optional[int] = None):
35
+ group_size: Optional[int] = None,
36
+ actorder: Optional[ActivationOrdering] = None):
37
38
self.pack_factor = 32 // num_bits
39
self.strategy = strategy
40
self.group_size = -1 if group_size is None else group_size
41
+ self.has_g_idx = actorder == ActivationOrdering.GROUP
42
43
if self.group_size == -1 and self.strategy != "channel":
44
raise ValueError("Marlin kernels require group quantization or "
@@ -64,12 +70,10 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
70
output_size_per_partition = sum(output_partition_sizes)
71
72
# If group_size is -1, we are in channelwise case.
67
- channelwise = (self.group_size == -1)
73
group_size = self.group_size if self.group_size != -1 else input_size
74
row_parallel = (input_size != input_size_per_partition)
70
- # In the case of channelwise quantization, we need to replicate the
71
- # scales across all gpus.
72
- partition_scales = (row_parallel and not channelwise)
75
+ partition_scales = not marlin_repeat_scales_on_all_ranks(
76
+ self.has_g_idx, self.group_size, row_parallel)
77
78
verify_marlin_supports_shape(
79
output_size_per_partition=output_size_per_partition,
@@ -123,6 +127,16 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
127
layer.register_parameter("weight_scale", weight_scale)
128
layer.register_parameter("weight_shape", weight_shape)
129
130
+ # group index (for activation reordering)
131
+ if self.has_g_idx:
132
+ weight_g_idx = RowvLLMParameter(data=torch.empty(
133
+ input_size_per_partition,
134
+ dtype=torch.int32,
135
+ ),
136
+ input_dim=0,
137
+ weight_loader=weight_loader)
138
+ layer.register_parameter("weight_g_idx", weight_g_idx)
139
+
140
layer.input_size_per_partition = input_size_per_partition
141
layer.output_size_per_partition = output_size_per_partition
142
layer.input_size = input_size
@@ -137,9 +151,14 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
151
layer.workspace = marlin_make_workspace(
152
layer.output_size_per_partition, device)
153
140
- # Act-order not supported in compressed-tensors yet, so set to empty.
141
- layer.g_idx = marlin_make_empty_g_idx(device)
142
- layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
154
+ # Handle sorting for activation reordering if needed.
155
+ if self.has_g_idx:
156
+ g_idx, g_idx_sort_indices = marlin_sort_g_idx(layer.weight_g_idx)
157
+ layer.g_idx_sort_indices = g_idx_sort_indices
158
+ replace_tensor(layer, "weight_g_idx", g_idx)
159
+ else:
160
+ layer.weight_g_idx = marlin_make_empty_g_idx(device)
161
+ layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
162
163
# No zero-point
164
layer.weight_zp = marlin_make_empty_g_idx(device)
@@ -159,9 +178,11 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
178
replace_tensor(layer, "weight_packed", marlin_qweight)
179
180
# Permute scales from compressed-tensors format to marlin format.
181
+ # scale is required on all partitions if activation reordering
182
marlin_scales = marlin_permute_scales(
183
layer.weight_scale,
164
- size_k=layer.input_size_per_partition,
184
+ size_k=(layer.input_size
185
+ if self.has_g_idx else layer.input_size_per_partition),
186
size_n=layer.output_size_per_partition,
187
group_size=layer.group_size)
188
replace_tensor(layer, "weight_scale", marlin_scales)
@@ -174,7 +195,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
195
weight=layer.weight_packed,
196
weight_scale=layer.weight_scale,
197
weight_zp=layer.weight_zp,
177
- g_idx=layer.g_idx,
198
+ g_idx=layer.weight_g_idx,
199
g_idx_sort_indices=layer.g_idx_sort_indices,
200
workspace=layer.workspace,
201
wtype=self.quant_type,
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+28
-2
@@ -1,8 +1,8 @@
1
import re
2
from enum import Enum
3
-from typing import Any, Dict, Iterable, Optional
3
+from typing import Any, Dict, Iterable, Optional, Union
4
5
-from pydantic import BaseModel, Field
5
+from pydantic import BaseModel, Field, field_validator
6
from torch.nn import Module
7
8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -40,6 +40,19 @@ class QuantizationStrategy(str, Enum):
40
TOKEN = "token"
41
42
43
+class ActivationOrdering(str, Enum):
44
+ """
45
+ Enum storing strategies for activation ordering
46
+
47
+ Group: reorder groups and weight\n
48
+ Weight: only reorder weight, not groups. Slightly lower latency and
49
+ accuracy compared to group actorder\n
50
+ """
51
+
52
+ GROUP = "group"
53
+ WEIGHT = "weight"
54
+
55
+
56
class QuantizationArgs(BaseModel):
57
"""
58
User facing arguments used to define a quantization config
@@ -58,6 +71,8 @@ class QuantizationArgs(BaseModel):
71
observed with every sample. Defaults to False for static
72
quantization. Note that enabling dynamic quantization
73
will change the default observer to a memoryless one
74
+ :param actorder: whether to apply group quantization in decreasing order of
75
+ activation. Defaults to None for arbitrary ordering
76
"""
77
78
num_bits: int = 8
@@ -67,6 +82,7 @@ class QuantizationArgs(BaseModel):
82
strategy: Optional[QuantizationStrategy] = None
83
block_structure: Optional[str] = None
84
dynamic: bool = False
85
+ actorder: Union[ActivationOrdering, bool, None] = None
86
observer: str = Field(
87
default="minmax",
88
description=("The class to use to compute the quantization param - "
@@ -79,6 +95,16 @@ class QuantizationArgs(BaseModel):
95
"Observers constructor excluding quantization range or symmetry"),
96
)
97
98
+ @field_validator("actorder", mode="before")
99
+ def validate_actorder(cls, value) -> Optional[ActivationOrdering]:
100
+ if isinstance(value, bool):
101
+ return ActivationOrdering.GROUP if value else None
102
+
103
+ if isinstance(value, str):
104
+ return ActivationOrdering(value.lower())
105
+
106
+ return value
107
+
108
109
def is_activation_quantization_format(format: str) -> bool:
110
_ACTIVATION_QUANTIZATION_FORMATS = [