A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Kernel] `w4a16` support for `compressed-tensors` (#5385)
Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com>
Dipika Sikka committed
Jun 13, 2024 at 10:19 UTC
c2637a613b6140dc16fecd5a1b0f5a9e1d0932ff
4 files changed
+230
-10
tests/quantization/test_compressed_tensors.py
+25
-2
@@ -3,12 +3,13 @@
3
Run `pytest tests/quantization/test_compressed_tensors.py`.
4
"""
5
6
+import pytest
7
import torch
8
9
from vllm import SamplingParams
10
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
10
- CompressedTensorsLinearMethod, CompressedTensorsW8A8DynamicToken,
11
- CompressedTensorsW8A8StaticTensor)
11
+ CompressedTensorsLinearMethod, CompressedTensorsW4A16,
12
+ CompressedTensorsW8A8DynamicToken, CompressedTensorsW8A8StaticTensor)
13
14
15
def test_compressed_tensors_w8a8_static_setup(vllm_runner):
@@ -60,3 +61,25 @@ def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner):
61
assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
62
assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8DynamicToken)
63
assert qkv_proj.weight.dtype is torch.int8
64
+
65
+
66
+@pytest.mark.parametrize("w4a16_args", [
67
+ ("nm-testing/tinyllama-oneshot-w4a16-channel-v2", "channel", None),
68
+ ("nm-testing/tinyllama-oneshot-w4a16-group128-v2", "group", 128),
69
+])
70
+def test_compressed_tensors_w4a16(vllm_runner, w4a16_args):
71
+ model, strategy, group = w4a16_args
72
+ with vllm_runner(model) as llm:
73
+ model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
74
+ layer = model.model.layers[0]
75
+
76
+ qkv_proj = layer.self_attn.qkv_proj
77
+ assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
78
+ assert isinstance(qkv_proj.scheme, CompressedTensorsW4A16)
79
+
80
+ assert qkv_proj.scheme.strategy == strategy
81
+ assert qkv_proj.scheme.group_size == group
82
+
83
+ assert qkv_proj.weight_packed.dtype is torch.int32
84
+ assert qkv_proj.weight_scale.dtype is torch.float16
85
+ assert qkv_proj.weight_packed.pack_factor == 8
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+36
-8
@@ -7,8 +7,8 @@ from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
7
from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
8
QuantizationConfig)
9
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10
- CompressedTensorsScheme, CompressedTensorsW8A8DynamicToken,
11
- CompressedTensorsW8A8StaticTensor)
10
+ CompressedTensorsScheme, CompressedTensorsW4A16,
11
+ CompressedTensorsW8A8DynamicToken, CompressedTensorsW8A8StaticTensor)
12
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
13
QuantizationArgs, QuantizationStrategy, find_first_name_or_class_match)
14
@@ -47,16 +47,27 @@ class CompressedTensorsConfig(QuantizationConfig):
47
layer_quant_details: Dict[str, Any] = dict()
48
ignore: List[str] = config.get("ignore", None)
49
50
+ # The quant_config has multiple config_groups, each containing
51
+ # an input_activations key with details about how the activations are
52
+ # quantized, a weights key indicating how the weights are quantized,
53
+ # and a list of targets under the `targets` key, dictating which
54
+ # layers are impacted by the quantization details. The quantization
55
+ # details follow the structure defined by the QuantizationArgs
56
+ # pydantic model, which is used to verify the structure of the
57
+ # quant_config and also store the details for later use.
58
for key, quant_config in config["config_groups"].items():
59
targets = quant_config.get("targets")
60
for target in targets:
61
layer_quant_details[target] = {}
62
layer_quant_details[target][
55
- "weight"] = QuantizationArgs.parse_obj(
63
+ "weights"] = QuantizationArgs.parse_obj(
64
quant_config.get("weights"))
57
- layer_quant_details[target][
58
- "input"] = QuantizationArgs.parse_obj(
59
- quant_config.get("input_activations"))
65
+ try:
66
+ layer_quant_details[target][
67
+ "input_activations"] = QuantizationArgs.parse_obj(
68
+ quant_config.get("input_activations"))
69
+ except Exception:
70
+ layer_quant_details[target]["input_activations"] = None
71
72
return cls(layer_quant_details=layer_quant_details, ignore=ignore)
73
@@ -86,8 +97,23 @@ class CompressedTensorsConfig(QuantizationConfig):
97
98
return is_8_bits and is_token_tensor and is_symmetric and is_dynamic
99
100
+ def _is_w4a16(self, weight_quant: BaseModel,
101
+ input_quant: BaseModel) -> bool:
102
+ input_quant_none = input_quant is None
103
+ is_4_bits = weight_quant.num_bits == 4
104
+ is_symmetric = weight_quant.symmetric
105
+ is_static = not weight_quant.dynamic
106
+
107
+ return is_4_bits and input_quant_none and is_symmetric and is_static
108
+
109
def _get_schema(self, weight_quant: BaseModel,
110
input_quant: BaseModel) -> "CompressedTensorsScheme":
111
+
112
+ if self._is_w4a16(weight_quant, input_quant):
113
+ return CompressedTensorsW4A16(num_bits=weight_quant.num_bits,
114
+ strategy=weight_quant.strategy,
115
+ group_size=weight_quant.group_size)
116
+
117
if self._is_static_tensor_w8a8(weight_quant, input_quant):
118
return CompressedTensorsW8A8StaticTensor()
119
@@ -113,8 +139,9 @@ class CompressedTensorsConfig(QuantizationConfig):
139
raise ValueError(
140
f"Could not find quantization details for {layer}.")
141
116
- return self._get_schema(weight_quant=layer_quant_details["weight"],
117
- input_quant=layer_quant_details["input"])
142
+ return self._get_schema(
143
+ weight_quant=layer_quant_details["weights"],
144
+ input_quant=layer_quant_details["input_activations"])
145
146
147
class CompressedTensorsLinearMethod(LinearMethodBase):
@@ -140,6 +167,7 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
167
layer=layer,
168
input_size_per_partition=input_size_per_partition,
169
output_partition_sizes=output_partition_sizes,
170
+ input_size=input_size,
171
output_size=output_size,
172
params_dtype=params_dtype,
173
weight_loader=weight_loader)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
+1
@@ -1,6 +1,7 @@
1
from .compressed_tensors_scheme import CompressedTensorsScheme # noqa: F401
2
from .compressed_tensors_unquantized import ( # noqa: F401
3
CompressedTensorsUnquantized)
4
+from .compressed_tensors_w4a16 import CompressedTensorsW4A16 # noqa: F401
5
from .compressed_tensors_w8a8_dynamictoken import ( # noqa: F401, E501
6
CompressedTensorsW8A8DynamicToken)
7
from .compressed_tensors_w8a8_statictensor import ( # noqa: F401, E501
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16.py
new
+168
@@ -0,0 +1,168 @@
1
+from typing import Callable, List, Optional
2
+
3
+import torch
4
+from torch.nn import Parameter
5
+
6
+from vllm import _custom_ops as ops
7
+from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
+ CompressedTensorsScheme)
9
+from vllm.model_executor.layers.quantization.gptq_marlin import (
10
+ GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, GPTQMarlinState,
11
+ marlin_permute_scales)
12
+from vllm.model_executor.utils import set_weight_attrs
13
+
14
+__all__ = ["CompressedTensorsW4A16"]
15
+
16
+
17
+class CompressedTensorsW4A16(CompressedTensorsScheme):
18
+
19
+ def __init__(self,
20
+ strategy: str,
21
+ num_bits: int,
22
+ group_size: Optional[int] = None):
23
+ self.num_bits = num_bits
24
+ self.strategy = strategy
25
+ self.group_size = group_size
26
+
27
+ if self.strategy == "group" and self.group_size is None:
28
+ raise ValueError(
29
+ "group_size must be given when using strategy group")
30
+
31
+ def create_weights(self, layer: torch.nn.Module, input_size: int,
32
+ output_partition_sizes: List[int],
33
+ input_size_per_partition: int,
34
+ params_dtype: torch.dtype, weight_loader: Callable,
35
+ **kwargs):
36
+
37
+ pack_factor = 32 // self.num_bits
38
+ output_size_per_partition = sum(output_partition_sizes)
39
+
40
+ if self.group_size is not None:
41
+ group_size = self.group_size
42
+ else:
43
+ group_size = input_size
44
+
45
+ weight_scale_dim = None
46
+ scales_and_zp_size = input_size // group_size
47
+
48
+ if (input_size != input_size_per_partition
49
+ and self.group_size is not None):
50
+ weight_scale_dim = 1
51
+ scales_and_zp_size = input_size_per_partition // group_size
52
+
53
+ weight = Parameter(
54
+ torch.empty(
55
+ output_size_per_partition,
56
+ input_size_per_partition // pack_factor,
57
+ dtype=torch.int32,
58
+ ),
59
+ requires_grad=False,
60
+ )
61
+
62
+ set_weight_attrs(
63
+ weight, {
64
+ "input_dim": 1,
65
+ "output_dim": 0,
66
+ "packed_dim": 1,
67
+ "pack_factor": pack_factor
68
+ })
69
+ set_weight_attrs(weight, {"weight_loader": weight_loader})
70
+
71
+ layer.register_parameter("weight_packed", weight)
72
+
73
+ weight_scale = Parameter(
74
+ torch.empty(
75
+ output_size_per_partition,
76
+ scales_and_zp_size,
77
+ dtype=params_dtype,
78
+ ),
79
+ requires_grad=False,
80
+ )
81
+
82
+ set_weight_attrs(weight_scale, {"weight_loader": weight_loader})
83
+ set_weight_attrs(weight_scale, {
84
+ "input_dim": weight_scale_dim,
85
+ "output_dim": 0
86
+ })
87
+ layer.register_parameter("weight_scale", weight_scale)
88
+
89
+ # A 2D array defining the original shape of the weights
90
+ # before packing
91
+ weight_shape = Parameter(torch.empty(2, dtype=torch.int64),
92
+ requires_grad=False)
93
+
94
+ layer.register_parameter("weight_shape", weight_shape)
95
+ set_weight_attrs(weight_shape, {"weight_loader": weight_loader})
96
+
97
+ layer.input_size_per_partition = input_size_per_partition
98
+ layer.output_size_per_partition = output_size_per_partition
99
+
100
+ layer.input_size = input_size
101
+ layer.marlin_state = GPTQMarlinState.REPACK
102
+ layer.is_k_full = True
103
+ layer.group_size = group_size
104
+
105
+ max_workspace_size = (
106
+ output_size_per_partition //
107
+ GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL
108
+
109
+ workspace = torch.zeros(max_workspace_size,
110
+ dtype=torch.int,
111
+ requires_grad=False)
112
+ layer.workspace = workspace
113
+
114
+ def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
115
+ reshaped_x = x.reshape(-1, x.shape[-1])
116
+
117
+ size_m = reshaped_x.shape[0]
118
+ part_size_n = layer.output_size_per_partition
119
+ part_size_k = layer.input_size_per_partition
120
+
121
+ out_shape = x.shape[:-1] + (part_size_n, )
122
+
123
+ if layer.marlin_state == GPTQMarlinState.REPACK:
124
+ layer.marlin_state = GPTQMarlinState.READY
125
+
126
+ # Newly generated tensors need to replace existing tensors that are
127
+ # already registered as parameters by vLLM (and won't be freed)
128
+ def replace_tensor(name, new_t):
129
+ # It is important to use resize_() here since it ensures
130
+ # the same buffer is reused
131
+ getattr(layer, name).resize_(new_t.shape)
132
+ getattr(layer, name).copy_(new_t)
133
+ del new_t
134
+
135
+ cur_device = layer.weight_packed.device
136
+
137
+ # Reset g_idx related tensors
138
+ layer.g_idx = Parameter(torch.empty(0,
139
+ dtype=torch.int,
140
+ device=cur_device),
141
+ requires_grad=False)
142
+ layer.g_idx_sort_indices = Parameter(torch.empty(
143
+ 0, dtype=torch.int, device=cur_device),
144
+ requires_grad=False)
145
+
146
+ # Repack weights
147
+ marlin_qweight = ops.gptq_marlin_repack(
148
+ layer.weight_packed.t().contiguous(), layer.g_idx_sort_indices,
149
+ part_size_k, part_size_n, self.num_bits)
150
+
151
+ replace_tensor("weight_packed", marlin_qweight)
152
+
153
+ # Permute scales
154
+ scales_size_k = part_size_k
155
+ scales_size_n = part_size_n
156
+
157
+ marlin_scales = marlin_permute_scales(
158
+ layer.weight_scale.squeeze().t().contiguous(), scales_size_k,
159
+ scales_size_n, layer.group_size, self.num_bits)
160
+ replace_tensor("weight_scale", marlin_scales)
161
+
162
+ output = ops.gptq_marlin_gemm(reshaped_x, layer.weight_packed,
163
+ layer.weight_scale, layer.g_idx,
164
+ layer.g_idx_sort_indices,
165
+ layer.workspace, self.num_bits, size_m,
166
+ part_size_n, part_size_k,
167
+ layer.is_k_full)
168
+ return output.reshape(out_shape)