@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c7cb5c3335
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Misc] GPTQ Activation Ordering (#8135)

Kyle Sayers committed Sep 9, 2024 at 16:27 UTC c7cb5c333564cb00fc4f6a99d32c35e9ebc0f1ed
4 files changed +64 -15
tests/weight_loading/models.txt
+1
@@ -21,6 +21,7 @@ compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
21 compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
22 compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
23 compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
24 +compressed-tensors, nm-testing/TinyLlama-1.1B-Chat-v1.0-actorder-group, main
25 awq, casperhansen/mixtral-instruct-awq, main
26 awq_marlin, casperhansen/mixtral-instruct-awq, main
27 fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+2 -1
@@ -232,7 +232,8 @@ class CompressedTensorsConfig(QuantizationConfig):
232 return CompressedTensorsWNA16(
233 num_bits=weight_quant.num_bits,
234 strategy=weight_quant.strategy,
235 - group_size=weight_quant.group_size)
235 + group_size=weight_quant.group_size,
236 + actorder=weight_quant.actorder)
237
238 # Detect If Activation Quantization.
239 # TODO @dsikka: clean-up conditions
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+33 -12
@@ -5,14 +5,18 @@ import torch
5 from vllm import _custom_ops as ops
6 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7 CompressedTensorsScheme)
8 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
9 + ActivationOrdering)
10 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
11 apply_gptq_marlin_linear, marlin_make_empty_g_idx, marlin_make_workspace,
10 - marlin_permute_scales, replace_tensor, verify_marlin_supported,
12 + marlin_permute_scales, marlin_repeat_scales_on_all_ranks,
13 + marlin_sort_g_idx, replace_tensor, verify_marlin_supported,
14 verify_marlin_supports_shape)
15 from vllm.model_executor.parameter import (BasevLLMParameter,
16 ChannelQuantScaleParameter,
17 GroupQuantScaleParameter,
15 - PackedvLLMParameter)
18 + PackedvLLMParameter,
19 + RowvLLMParameter)
20 from vllm.scalar_type import scalar_types
21
22 __all__ = ["CompressedTensorsWNA16"]
@@ -28,11 +32,13 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
32 def __init__(self,
33 strategy: str,
34 num_bits: int,
31 - group_size: Optional[int] = None):
35 + group_size: Optional[int] = None,
36 + actorder: Optional[ActivationOrdering] = None):
37
38 self.pack_factor = 32 // num_bits
39 self.strategy = strategy
40 self.group_size = -1 if group_size is None else group_size
41 + self.has_g_idx = actorder == ActivationOrdering.GROUP
42
43 if self.group_size == -1 and self.strategy != "channel":
44 raise ValueError("Marlin kernels require group quantization or "
@@ -64,12 +70,10 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
70 output_size_per_partition = sum(output_partition_sizes)
71
72 # If group_size is -1, we are in channelwise case.
67 - channelwise = (self.group_size == -1)
73 group_size = self.group_size if self.group_size != -1 else input_size
74 row_parallel = (input_size != input_size_per_partition)
70 - # In the case of channelwise quantization, we need to replicate the
71 - # scales across all gpus.
72 - partition_scales = (row_parallel and not channelwise)
75 + partition_scales = not marlin_repeat_scales_on_all_ranks(
76 + self.has_g_idx, self.group_size, row_parallel)
77
78 verify_marlin_supports_shape(
79 output_size_per_partition=output_size_per_partition,
@@ -123,6 +127,16 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
127 layer.register_parameter("weight_scale", weight_scale)
128 layer.register_parameter("weight_shape", weight_shape)
129
130 + # group index (for activation reordering)
131 + if self.has_g_idx:
132 + weight_g_idx = RowvLLMParameter(data=torch.empty(
133 + input_size_per_partition,
134 + dtype=torch.int32,
135 + ),
136 + input_dim=0,
137 + weight_loader=weight_loader)
138 + layer.register_parameter("weight_g_idx", weight_g_idx)
139 +
140 layer.input_size_per_partition = input_size_per_partition
141 layer.output_size_per_partition = output_size_per_partition
142 layer.input_size = input_size
@@ -137,9 +151,14 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
151 layer.workspace = marlin_make_workspace(
152 layer.output_size_per_partition, device)
153
140 - # Act-order not supported in compressed-tensors yet, so set to empty.
141 - layer.g_idx = marlin_make_empty_g_idx(device)
142 - layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
154 + # Handle sorting for activation reordering if needed.
155 + if self.has_g_idx:
156 + g_idx, g_idx_sort_indices = marlin_sort_g_idx(layer.weight_g_idx)
157 + layer.g_idx_sort_indices = g_idx_sort_indices
158 + replace_tensor(layer, "weight_g_idx", g_idx)
159 + else:
160 + layer.weight_g_idx = marlin_make_empty_g_idx(device)
161 + layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
162
163 # No zero-point
164 layer.weight_zp = marlin_make_empty_g_idx(device)
@@ -159,9 +178,11 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
178 replace_tensor(layer, "weight_packed", marlin_qweight)
179
180 # Permute scales from compressed-tensors format to marlin format.
181 + # scale is required on all partitions if activation reordering
182 marlin_scales = marlin_permute_scales(
183 layer.weight_scale,
164 - size_k=layer.input_size_per_partition,
184 + size_k=(layer.input_size
185 + if self.has_g_idx else layer.input_size_per_partition),
186 size_n=layer.output_size_per_partition,
187 group_size=layer.group_size)
188 replace_tensor(layer, "weight_scale", marlin_scales)
@@ -174,7 +195,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
195 weight=layer.weight_packed,
196 weight_scale=layer.weight_scale,
197 weight_zp=layer.weight_zp,
177 - g_idx=layer.g_idx,
198 + g_idx=layer.weight_g_idx,
199 g_idx_sort_indices=layer.g_idx_sort_indices,
200 workspace=layer.workspace,
201 wtype=self.quant_type,
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+28 -2
@@ -1,8 +1,8 @@
1 import re
2 from enum import Enum
3 -from typing import Any, Dict, Iterable, Optional
3 +from typing import Any, Dict, Iterable, Optional, Union
4
5 -from pydantic import BaseModel, Field
5 +from pydantic import BaseModel, Field, field_validator
6 from torch.nn import Module
7
8 from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -40,6 +40,19 @@ class QuantizationStrategy(str, Enum):
40 TOKEN = "token"
41
42
43 +class ActivationOrdering(str, Enum):
44 + """
45 + Enum storing strategies for activation ordering
46 +
47 + Group: reorder groups and weight\n
48 + Weight: only reorder weight, not groups. Slightly lower latency and
49 + accuracy compared to group actorder\n
50 + """
51 +
52 + GROUP = "group"
53 + WEIGHT = "weight"
54 +
55 +
56 class QuantizationArgs(BaseModel):
57 """
58 User facing arguments used to define a quantization config
@@ -58,6 +71,8 @@ class QuantizationArgs(BaseModel):
71 observed with every sample. Defaults to False for static
72 quantization. Note that enabling dynamic quantization
73 will change the default observer to a memoryless one
74 + :param actorder: whether to apply group quantization in decreasing order of
75 + activation. Defaults to None for arbitrary ordering
76 """
77
78 num_bits: int = 8
@@ -67,6 +82,7 @@ class QuantizationArgs(BaseModel):
82 strategy: Optional[QuantizationStrategy] = None
83 block_structure: Optional[str] = None
84 dynamic: bool = False
85 + actorder: Union[ActivationOrdering, bool, None] = None
86 observer: str = Field(
87 default="minmax",
88 description=("The class to use to compute the quantization param - "
@@ -79,6 +95,16 @@ class QuantizationArgs(BaseModel):
95 "Observers constructor excluding quantization range or symmetry"),
96 )
97
98 + @field_validator("actorder", mode="before")
99 + def validate_actorder(cls, value) -> Optional[ActivationOrdering]:
100 + if isinstance(value, bool):
101 + return ActivationOrdering.GROUP if value else None
102 +
103 + if isinstance(value, str):
104 + return ActivationOrdering(value.lower())
105 +
106 + return value
107 +
108
109 def is_activation_quantization_format(format: str) -> bool:
110 _ACTIVATION_QUANTIZATION_FORMATS = [