A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Kernel] AQ AZP 4/4: Integrate asymmetric quantization to linear method (#7271)
Luka Govedič committed
Sep 27, 2024 at 14:25 UTC
172d1cd27634e9e7adc9cb9feac73552cfae1b24
7 files changed
+124
-21
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-INT8-compressed-tensors-asym.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Asym-Per-Token-Test -b "auto" -l 250 -f 5 -t 1
2
+model_name: "nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Asym-Per-Token-Test"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.764
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.764
10
+limit: 250
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1
@@ -1,6 +1,7 @@
1
Meta-Llama-3-8B-Instruct.yaml
2
Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
3
Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
4
+Meta-Llama-3-8B-Instruct-INT8-compressed-tensors-asym.yaml
5
Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
6
Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
7
Minitron-4B-Base-FP8.yaml
.buildkite/lm-eval-harness/test_lm_eval_correctness.py
+6
-1
@@ -49,10 +49,15 @@ def test_lm_eval_correctness():
49
results = launch_lm_eval(eval_config)
50
51
# Confirm scores match ground truth.
52
+ success = True
53
for task in eval_config["tasks"]:
54
for metric in task["metrics"]:
55
ground_truth = metric["value"]
56
measured_value = results["results"][task["name"]][metric["name"]]
57
print(f'{task["name"]} | {metric["name"]}: '
58
f'ground_truth={ground_truth} | measured={measured_value}')
58
- assert numpy.isclose(ground_truth, measured_value, rtol=RTOL)
59
+ success = success and numpy.isclose(
60
+ ground_truth, measured_value, rtol=RTOL)
61
+
62
+ # Assert at the end, print all scores even on failure for debugging.
63
+ assert success
tests/quantization/test_compressed_tensors.py
+27
-9
@@ -2,6 +2,7 @@
2
3
Run `pytest tests/quantization/test_compressed_tensors.py`.
4
"""
5
+from typing import Optional
6
7
import pytest
8
import torch
@@ -14,14 +15,16 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
15
QuantizationType)
16
17
17
-@pytest.mark.parametrize("model_args", [
18
- ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", "tensor",
19
- QuantizationType.INT, 2560),
20
- ("nm-testing/tinyllama-oneshot-w8-channel-a8-tensor", "channel",
21
- QuantizationType.INT, 2560),
22
-])
18
+@pytest.mark.parametrize(
19
+ "model_args",
20
+ [("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", "tensor",
21
+ QuantizationType.INT, 2560, True),
22
+ ("nm-testing/tinyllama-oneshot-w8-channel-a8-tensor", "channel",
23
+ QuantizationType.INT, 2560, True),
24
+ ("nm-testing/asym-w8w8-int8-static-per-tensor-tiny-llama", "tensor",
25
+ QuantizationType.INT, 2560, False)])
26
def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
24
- model_path, strategy, quant_type, shape_0 = model_args
27
+ model_path, strategy, quant_type, shape_0, is_symmetric = model_args
28
with vllm_runner(model_path, enforce_eager=True) as llm:
29
model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
30
layer = model.model.layers[0]
@@ -31,6 +34,18 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
34
gate_up_proj = layer.mlp.gate_up_proj
35
down_proj = layer.mlp.down_proj
36
37
+ # assert zp for symmetric and asymmetric cases
38
+ def zp_valid(zp: Optional[torch.Tensor]):
39
+ if is_symmetric:
40
+ return zp is None
41
+
42
+ return zp is not None and zp.dtype is torch.int32
43
+
44
+ assert zp_valid(qkv_proj.input_zero_point)
45
+ assert zp_valid(o_proj.input_zero_point)
46
+ assert zp_valid(gate_up_proj.input_zero_point)
47
+ assert zp_valid(down_proj.input_zero_point)
48
+
49
assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
50
assert isinstance(o_proj.quant_method, CompressedTensorsLinearMethod)
51
assert isinstance(gate_up_proj.quant_method,
@@ -69,9 +84,12 @@ def test_compressed_tensors_no_enforce_eager(vllm_runner):
84
85
@pytest.mark.parametrize("model_args", [
86
("nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2", "tensor"),
87
+ ("nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2-asym", "tensor"),
88
("nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2", "channel"),
89
+ ("nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2-asym",
90
+ "channel"),
91
])
74
-def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner, model_args):
92
+def test_compressed_tensors_w8a8_dynamic_per_token(vllm_runner, model_args):
93
model_path, strategy = model_args
94
with vllm_runner(model_path, dtype=torch.float16) as llm:
95
model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
@@ -160,4 +178,4 @@ def test_compressed_tensors_kv_cache(vllm_runner):
178
model_path = "nm-testing/TinyLlama-1.1B-compressed-tensors-kv-cache-scheme"
179
with vllm_runner(model_path, kv_cache_dtype="fp8") as llm:
180
output = llm.generate_greedy("Hello world!", max_tokens=20)
163
- assert output
\ No newline at end of file
181
+ assert output
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+10
-6
@@ -138,10 +138,11 @@ class CompressedTensorsConfig(QuantizationConfig):
138
or weight_quant.strategy == QuantizationStrategy.CHANNEL.value)
139
is_tensor = (weight_strategy and input_quant.strategy
140
== QuantizationStrategy.TENSOR.value)
141
- is_symmetric = weight_quant.symmetric and input_quant.symmetric
141
is_static = not weight_quant.dynamic and not input_quant.dynamic
142
144
- return is_8_bits and is_tensor and is_symmetric and is_static
143
+ # Both symmetric and asymmetric input quantization supported.
144
+ # Only symmetric weight quantization supported.
145
+ return is_8_bits and is_tensor and weight_quant.symmetric and is_static
146
147
def _is_dynamic_token_w8a8(self, weight_quant: BaseModel,
148
input_quant: BaseModel) -> bool:
@@ -151,10 +152,11 @@ class CompressedTensorsConfig(QuantizationConfig):
152
or weight_quant.strategy == QuantizationStrategy.CHANNEL.value)
153
is_token = (weight_strategy and input_quant.strategy
154
== QuantizationStrategy.TOKEN.value)
154
- is_symmetric = weight_quant.symmetric and input_quant.symmetric
155
is_dynamic = not weight_quant.dynamic and input_quant.dynamic
156
157
- return is_8_bits and is_token and is_symmetric and is_dynamic
157
+ # Both symmetric and asymmetric input quantization supported.
158
+ # Only symmetric weight quantization supported.
159
+ return is_8_bits and is_token and weight_quant.symmetric and is_dynamic
160
161
def _is_fp8_w8a8(self, weight_quant: BaseModel,
162
input_quant: BaseModel) -> bool:
@@ -265,12 +267,14 @@ class CompressedTensorsConfig(QuantizationConfig):
267
if self._is_static_tensor_w8a8(weight_quant, input_quant):
268
return CompressedTensorsW8A8Int8(
269
strategy=weight_quant.strategy,
268
- is_static_input_scheme=True)
270
+ is_static_input_scheme=True,
271
+ input_symmetric=input_quant.symmetric)
272
273
if self._is_dynamic_token_w8a8(weight_quant, input_quant):
274
return CompressedTensorsW8A8Int8(
275
strategy=weight_quant.strategy,
273
- is_static_input_scheme=False)
276
+ is_static_input_scheme=False,
277
+ input_symmetric=input_quant.symmetric)
278
279
raise NotImplementedError(
280
"No compressed-tensors compatible scheme was found.")
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+52
-3
@@ -3,6 +3,7 @@ from typing import Callable, List, Optional
3
import torch
4
from torch.nn import Parameter
5
6
+from vllm.logger import init_logger
7
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
CompressedTensorsScheme)
9
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
@@ -14,12 +15,16 @@ from vllm.model_executor.parameter import (BasevLLMParameter,
15
ModelWeightParameter,
16
PerTensorScaleParameter)
17
18
+logger = init_logger(__name__)
19
+
20
21
class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
22
20
- def __init__(self, strategy: str, is_static_input_scheme: bool):
23
+ def __init__(self, strategy: str, is_static_input_scheme: bool,
24
+ input_symmetric: bool):
25
self.strategy = strategy
26
self.is_static_input_scheme = is_static_input_scheme
27
+ self.input_symmetric = input_symmetric
28
29
@classmethod
30
def get_min_capability(cls) -> int:
@@ -46,10 +51,43 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
51
requires_grad=False)
52
# INPUT SCALE
53
if self.is_static_input_scheme:
49
- layer.input_scale = Parameter(layer.input_scale.max(),
50
- requires_grad=False)
54
+ if self.input_symmetric:
55
+ layer.input_scale = Parameter(layer.input_scale.max(),
56
+ requires_grad=False)
57
+ layer.input_zero_point = None
58
+ else:
59
+ # reconstruct the ranges
60
+ int8_traits = torch.iinfo(torch.int8)
61
+ azps = layer.input_zero_point.to(dtype=torch.int32)
62
+ range_max = (layer.input_scale *
63
+ (int8_traits.max - azps)).max()
64
+ range_min = (layer.input_scale *
65
+ (int8_traits.min - azps)).min()
66
+
67
+ scale = (range_max - range_min) / (int8_traits.max -
68
+ int8_traits.min)
69
+ layer.input_scale = Parameter(scale, requires_grad=False)
70
+
71
+ # AZP loaded as int8 but used as int32
72
+ azp = (int8_traits.min -
73
+ range_min / scale).to(dtype=torch.int32)
74
+ layer.input_zero_point = Parameter(azp, requires_grad=False)
75
+
76
else:
77
layer.input_scale = None
78
+ layer.input_zero_point = None
79
+
80
+ # azp_adj is the AZP adjustment term, used to account for weights.
81
+ # It does not depend on scales or azp, so it is the same for
82
+ # static and dynamic quantization.
83
+ # For more details, see csrc/quantization/cutlass_w8a8/Epilogues.md
84
+ # https://github.com/vllm-project/vllm/blob/8d59dbb00044a588cab96bcdc028006ed922eb06/csrc/quantization/cutlass_w8a8/Epilogues.md
85
+ if not self.input_symmetric:
86
+ layer.azp_adj = layer.weight.sum(dim=0,
87
+ keepdim=True,
88
+ dtype=torch.int32)
89
+ else:
90
+ layer.azp_adj = None
91
92
def create_weights(self, layer: torch.nn.Module,
93
output_partition_sizes: List[int],
@@ -90,6 +128,15 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
128
weight_loader=weight_loader)
129
layer.register_parameter("input_scale", input_scale)
130
131
+ if not self.input_symmetric:
132
+ # Note: compressed-tensors stores the zp using the same dtype
133
+ # as the weights
134
+ # AZP loaded as int8 but used as int32
135
+ input_zero_point = BasevLLMParameter(
136
+ data=torch.empty(1, dtype=torch.int8),
137
+ weight_loader=weight_loader)
138
+ layer.register_parameter("input_zero_point", input_zero_point)
139
+
140
def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
141
bias: Optional[torch.Tensor]) -> torch.Tensor:
142
@@ -97,4 +144,6 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
144
weight=layer.weight,
145
weight_scale=layer.weight_scale,
146
input_scale=layer.input_scale,
147
+ input_zero_point=layer.input_zero_point,
148
+ azp_adj=layer.azp_adj,
149
bias=bias)
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+17
-2
@@ -191,13 +191,28 @@ def apply_int8_linear(
191
weight: torch.Tensor,
192
weight_scale: torch.Tensor,
193
input_scale: Optional[torch.Tensor] = None,
194
+ input_zero_point: Optional[torch.Tensor] = None,
195
+ azp_adj: Optional[torch.Tensor] = None,
196
bias: Optional[torch.Tensor] = None,
197
):
198
# ops.scaled_int8_quant supports both dynamic and static quant.
199
# * dynamic, layer.input_scale is None and x_scale computed from x.
200
# * static, layer.input_scale is scalar and x_scale is input_scale.
199
- x_q, x_scale, _ = ops.scaled_int8_quant(input, input_scale)
200
-
201
+ symmetric = azp_adj is None
202
+ x_q, x_scale, x_zp = ops.scaled_int8_quant(input,
203
+ input_scale,
204
+ input_zero_point,
205
+ symmetric=symmetric)
206
+
207
+ if x_zp is not None:
208
+ return ops.cutlass_scaled_mm_azp(x_q,
209
+ weight,
210
+ scale_a=x_scale,
211
+ scale_b=weight_scale,
212
+ out_dtype=input.dtype,
213
+ azp_adj=azp_adj,
214
+ azp=x_zp,
215
+ bias=bias)
216
return ops.cutlass_scaled_mm(x_q,
217
weight,
218
scale_a=x_scale,