@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bf2ddc6610
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[bugfix] Fix static asymmetric quantization case (#10334)

Signed-off-by: Daniël de Kok <me@danieldk.eu> Signed-off-by: luka <luka@neuralmagic.com> Co-authored-by: Daniël de Kok <me@danieldk.eu>

Luka Govedič committed Nov 14, 2024 at 20:35 UTC bf2ddc6610094524a61e90441e579d502c7dee06
5 files changed +58 -15
tests/kernels/test_int8_quant.py
+10 -9
@@ -86,10 +86,7 @@ def test_dynamic_scaled_int8_azp_quant(num_tokens: int, hidden_size: int,
86 assert torch_out.min() >= int8_traits.min and torch_out.max(
87 ) <= int8_traits.max
88
89 - ops_out = torch.empty_like(x, dtype=torch.int8)
90 - scales_out = torch.empty_like(scales, dtype=torch.float32)
91 - azp_out = torch.empty_like(azps, dtype=torch.int32)
92 - torch.ops._C.dynamic_scaled_int8_quant(ops_out, x, scales_out, azp_out)
89 + ops_out, scales_out, azp_out = scaled_int8_quant(x, symmetric=False)
90
91 if (not torch.allclose(scales_out, scales)):
92 print(torch.argmax(torch.abs(scales_out - scales)))
@@ -119,7 +116,8 @@ def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
116
117 out1 = (x / scale_arg).round().clamp(int8_traits.min,
118 int8_traits.max).to(torch.int8)
122 - out2, _, _ = scaled_int8_quant(x, scale_arg)
119 + out2, scale2, _ = scaled_int8_quant(x, scale_arg)
120 + assert scale2 is scale_arg
121
122 # big atol to account for rounding errors
123 torch.testing.assert_close(out1, out2, atol=1, rtol=0.0)
@@ -145,11 +143,15 @@ def test_static_scaled_int8_azp_quant(num_tokens: int, hidden_size: int,
143
144 out1 = ((x / scale).round() + azp).clamp(int8_traits.min,
145 int8_traits.max).to(torch.int8)
148 - out2 = torch.empty_like(x, dtype=torch.int8)
146 scale_arg = torch.tensor([scale], dtype=torch.float32, device="cuda")
147 azp_arg = torch.tensor([azp], dtype=torch.int32, device="cuda")
148
152 - torch.ops._C.static_scaled_int8_quant(out2, x, scale_arg, azp_arg)
149 + out2, scale2, azp2 = scaled_int8_quant(x,
150 + scale_arg,
151 + azp_arg,
152 + symmetric=False)
153 + assert scale2 is scale_arg
154 + assert azp2 is azp_arg
155
156 # big atol to account for rounding errors
157 torch.testing.assert_close(out1, out2, atol=1, rtol=0.0)
@@ -184,6 +186,5 @@ def test_static_scaled_int8_azp_quant_saturating_cast(is_max: bool) -> None:
186 val_i8 = int8_traits.max if is_max else int8_traits.min
187 expected = torch.full((1, 5), val_i8, dtype=torch.int8, device="cuda")
188
187 - out = torch.empty_like(expected)
188 - torch.ops._C.static_scaled_int8_quant(out, x, scale, azp)
189 + out, _, _ = scaled_int8_quant(x, scale, azp, symmetric=False)
190 torch.testing.assert_close(expected, out, atol=0, rtol=0)
tests/quantization/test_compressed_tensors.py
+30
@@ -8,6 +8,7 @@ import pytest
8 import torch
9 from compressed_tensors.quantization import QuantizationType
10
11 +from tests.models.utils import check_logprobs_close
12 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
13 CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
14 CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
@@ -74,6 +75,35 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
75 assert output
76
77
78 +@pytest.mark.parametrize(
79 + "model_path",
80 + [
81 + "neuralmagic/Llama-3.2-1B-quantized.w8a8"
82 + # TODO static & asymmetric
83 + ])
84 +@pytest.mark.parametrize("max_tokens", [32])
85 +@pytest.mark.parametrize("num_logprobs", [10])
86 +def test_compressed_tensors_w8a8_logprobs(hf_runner, vllm_runner,
87 + example_prompts, model_path,
88 + max_tokens, num_logprobs):
89 + dtype = "bfloat16"
90 +
91 + with hf_runner(model_path, dtype=dtype) as hf_model:
92 + hf_outputs = hf_model.generate_greedy_logprobs_limit(
93 + example_prompts, max_tokens, num_logprobs)
94 +
95 + with vllm_runner(model_path, dtype=dtype) as vllm_model:
96 + vllm_outputs = vllm_model.generate_greedy_logprobs(
97 + example_prompts, max_tokens, num_logprobs)
98 +
99 + check_logprobs_close(
100 + outputs_0_lst=hf_outputs,
101 + outputs_1_lst=vllm_outputs,
102 + name_0="hf",
103 + name_1="vllm",
104 + )
105 +
106 +
107 def test_compressed_tensors_no_enforce_eager(vllm_runner):
108 model_path = "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change"
109 with vllm_runner(model_path) as llm:
vllm/_custom_ops.py
+7 -1
@@ -510,10 +510,16 @@ def cutlass_scaled_mm_azp(a: torch.Tensor,
510 azp_adj: torch.Tensor,
511 azp: Optional[torch.Tensor] = None,
512 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
513 + """
514 + :param azp_adj: In the per-tensor case, this should include the azp.
515 + Always per-channel.
516 + :param azp: Only set in the per-token case. Per-token if set.
517 + """
518 assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
519 assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
520 assert bias is None or bias.numel(
521 ) == b.shape[1] and bias.dtype == out_dtype
522 + assert azp is None or azp.numel() == a.shape[0]
523
524 m = a.shape[0]
525 n = b.shape[1]
@@ -735,7 +741,7 @@ def scaled_int8_quant(
741 azp is
742 None), "azp must only be provided for asymmetric quantization."
743 torch.ops._C.static_scaled_int8_quant(output, input, scale, azp)
738 - return output, scale, None
744 + return output, scale, azp
745
746 # dynamic-per-token quantization.
747 input_scales = torch.empty((input.numel() // input.shape[-1], 1),
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+7 -4
@@ -82,9 +82,13 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
82 # For more details, see csrc/quantization/cutlass_w8a8/Epilogues.md
83 # https://github.com/vllm-project/vllm/blob/8d59dbb00044a588cab96bcdc028006ed922eb06/csrc/quantization/cutlass_w8a8/Epilogues.md
84 if not self.input_symmetric:
85 - layer.azp_adj = layer.weight.sum(dim=0,
86 - keepdim=True,
87 - dtype=torch.int32)
85 + azp_adj = layer.weight.sum(dim=0, keepdim=True, dtype=torch.int32)
86 + if self.is_static_input_scheme:
87 + # cutlass_w8a8 requires azp to be folded into azp_adj
88 + # in the per-tensor case
89 + azp_adj = layer.input_zero_point * azp_adj
90 +
91 + layer.azp_adj = azp_adj
92 else:
93 layer.azp_adj = None
94
@@ -138,7 +142,6 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
142
143 def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
144 bias: Optional[torch.Tensor]) -> torch.Tensor:
141 -
145 return apply_int8_linear(input=x,
146 weight=layer.weight,
147 weight_scale=layer.weight_scale,
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+4 -1
@@ -211,13 +211,16 @@ def apply_int8_linear(
211 symmetric=symmetric)
212
213 if x_zp is not None:
214 + # Currently, static is always per-tensor and dynamic is per-token
215 + static = input_zero_point is not None
216 + azp = None if static else x_zp
217 return ops.cutlass_scaled_mm_azp(x_q,
218 weight,
219 scale_a=x_scale,
220 scale_b=weight_scale,
221 out_dtype=input.dtype,
222 azp_adj=azp_adj,
220 - azp=x_zp,
223 + azp=azp,
224 bias=bias)
225 return ops.cutlass_scaled_mm(x_q,
226 weight,