@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5d73ae49d6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Kernel] AQ AZP 3/4: Asymmetric quantization kernels (#7270)

Luka Govedič committed Sep 16, 2024 at 14:52 UTC 5d73ae49d65394f8dbe46accd921fb21e8247b82
9 files changed +339 -57
csrc/cpu/quant.cpp
+6 -3
@@ -257,11 +257,13 @@ void int8_scaled_mm(torch::Tensor& c, // [M, OC], row-major
257 // static-per-tensor quantization.
258 void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
259 const torch::Tensor& input, // [..., hidden_size]
260 - const torch::Tensor& scale) {
260 + const torch::Tensor& scale,
261 + c10::optional<torch::Tensor> const& azp) {
262 CPU_KERNEL_GUARD_IN(static_scaled_int8_quant)
263 TORCH_CHECK(input.is_contiguous());
264 TORCH_CHECK(out.is_contiguous());
265 TORCH_CHECK(scale.numel() == 1);
266 + TORCH_CHECK(!azp.has_value(), "Zero point is not supported on CPU.");
267
268 const int hidden_size = input.size(-1);
269 const int num_tokens = input.numel() / hidden_size;
@@ -277,11 +279,12 @@ void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
279 void dynamic_scaled_int8_quant(
280 torch::Tensor& out, // [..., hidden_size]
281 const torch::Tensor& input, // [..., hidden_size]
280 - torch::Tensor& scale // [..., 1]
281 -) {
282 + torch::Tensor& scale, // [..., 1]
283 + c10::optional<torch::Tensor> const& azp) {
284 CPU_KERNEL_GUARD_IN(dynamic_scaled_int8_quant)
285 TORCH_CHECK(input.is_contiguous());
286 TORCH_CHECK(out.is_contiguous());
287 + TORCH_CHECK(!azp.has_value(), "Zero point is not supported on CPU.");
288
289 int const hidden_size = input.size(-1);
290 int const num_tokens = input.numel() / hidden_size;
csrc/cpu/torch_bindings.cpp
+5 -4
@@ -94,13 +94,14 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
94 #ifdef __AVX512F__
95 // Compute int8 quantized tensor for given scaling factor.
96 ops.def(
97 - "static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale) -> "
98 - "()");
97 + "static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale,"
98 + "Tensor? azp) -> ()");
99 ops.impl("static_scaled_int8_quant", torch::kCPU, &static_scaled_int8_quant);
100 +
101 // Compute int8 quantized tensor and scaling factor
102 ops.def(
102 - "dynamic_scaled_int8_quant(Tensor! out, Tensor input, Tensor! scale) -> "
103 - "()");
103 + "dynamic_scaled_int8_quant(Tensor! out, Tensor input, Tensor! scale, "
104 + "Tensor!? azp) -> ()");
105 ops.impl("dynamic_scaled_int8_quant", torch::kCPU,
106 &dynamic_scaled_int8_quant);
107 // W8A8 GEMM, supporting symmetric per-tensor or per-row/column
csrc/ops.h
+4 -2
@@ -184,10 +184,12 @@ torch::Tensor marlin_qqq_gemm(torch::Tensor const& a,
184 #endif
185
186 void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
187 - torch::Tensor const& scale);
187 + torch::Tensor const& scale,
188 + c10::optional<torch::Tensor> const& azp);
189
190 void dynamic_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
190 - torch::Tensor& scales);
191 + torch::Tensor& scales,
192 + c10::optional<torch::Tensor> const& azp);
193
194 torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
195 torch::Tensor b_gptq_qzeros,
csrc/quantization/compressed_tensors/int8_quant_kernels.cu
+160 -13
@@ -14,12 +14,17 @@
14
15 static inline __device__ int8_t float_to_int8_rn(float x) {
16 #ifdef USE_ROCM
17 - static const float i8_min =
17 + static constexpr auto i8_min =
18 static_cast<float>(std::numeric_limits<int8_t>::min());
19 - static const float i8_max =
19 + static constexpr auto i8_max =
20 static_cast<float>(std::numeric_limits<int8_t>::max());
21 - // round
21 +
22 + // To match the rounding mode of CUDA, we use nearbyint.
23 + // It uses the current rounding mode, which is always FE_TONEAREST on HIP.
24 + // If that changes in the future, we may need to set the rounding mode
25 + // explicitly, either at runtime or compile time.
26 float dst = std::nearbyint(x);
27 +
28 // saturate
29 dst = std::clamp(dst, i8_min, i8_max);
30 return static_cast<int8_t>(dst);
@@ -31,6 +36,59 @@ static inline __device__ int8_t float_to_int8_rn(float x) {
36 #endif
37 }
38
39 +static inline __device__ int32_t float_to_int32_rn(float x) {
40 +#ifdef USE_ROCM
41 + // int32_max is not exactly representable as float.
42 + // Therefore, we need to be careful and manually return int32_max on overflow.
43 + // For symmetry, we also do the same for int32_min, even though it is exactly
44 + // representable as float and the conversion should be exact.
45 + static constexpr auto i32_min = std::numeric_limits<int32_t>::min();
46 + static constexpr auto i32_min_f = static_cast<float>(i32_min);
47 + static constexpr auto i32_max = std::numeric_limits<int32_t>::max();
48 + static constexpr auto i32_max_f = static_cast<float>(i32_max);
49 +
50 + // To match the rounding mode of CUDA, we use nearbyint.
51 + // It uses the current rounding mode, which is always FE_TONEAREST on HIP.
52 + // If that changes in the future, we may need to set the rounding mode
53 + // explicitly, either at runtime or compile time.
54 + float dst = std::nearbyint(x);
55 +
56 + // saturate on the higher end.
57 + if (dst >= i32_max_f) {
58 + return i32_max;
59 + }
60 + // saturate on the lower end.
61 + if (dst <= i32_min_f) {
62 + return i32_min;
63 + }
64 +
65 + return static_cast<int32_t>(dst);
66 +#else
67 + // CUDA path
68 + uint32_t dst;
69 + asm volatile("cvt.rni.sat.s32.f32 %0, %1;" : "=r"(dst) : "f"(x));
70 + return reinterpret_cast<const int32_t&>(dst);
71 +#endif
72 +}
73 +
74 +static inline __device__ int8_t int32_to_int8(int32_t x) {
75 +#ifdef USE_ROCM
76 + static constexpr auto i8_min =
77 + static_cast<int32_t>(std::numeric_limits<int8_t>::min());
78 + static constexpr auto i8_max =
79 + static_cast<int32_t>(std::numeric_limits<int8_t>::max());
80 +
81 + // saturate
82 + int32_t dst = std::clamp(x, i8_min, i8_max);
83 + return static_cast<int8_t>(dst);
84 +#else
85 + // CUDA path
86 + uint32_t dst;
87 + asm volatile("cvt.sat.s8.s32 %0, %1;" : "=r"(dst) : "r"(x));
88 + return reinterpret_cast<const int8_t&>(dst);
89 +#endif
90 +}
91 +
92 namespace vllm {
93
94 template <typename scalar_t, typename scale_type>
@@ -47,6 +105,23 @@ __global__ void static_scaled_int8_quant_kernel(
105 }
106 }
107
108 +template <typename scalar_t, typename scale_type, typename azp_type>
109 +__global__ void static_scaled_int8_azp_quant_kernel(
110 + scalar_t const* __restrict__ input, int8_t* __restrict__ out,
111 + scale_type const* scale_ptr, azp_type const* azp_ptr,
112 + const int hidden_size) {
113 + int const tid = threadIdx.x;
114 + int const token_idx = blockIdx.x;
115 + scale_type const scale = *scale_ptr;
116 + azp_type const azp = *azp_ptr;
117 +
118 + for (int i = tid; i < hidden_size; i += blockDim.x) {
119 + auto const val = static_cast<float>(input[token_idx * hidden_size + i]);
120 + auto const quant_val = int32_to_int8(float_to_int32_rn(val / scale) + azp);
121 + out[token_idx * hidden_size + i] = quant_val;
122 + }
123 +}
124 +
125 template <typename scalar_t, typename scale_type>
126 __global__ void dynamic_scaled_int8_quant_kernel(
127 scalar_t const* __restrict__ input, int8_t* __restrict__ out,
@@ -80,14 +155,68 @@ __global__ void dynamic_scaled_int8_quant_kernel(
155 }
156 }
157
158 +template <typename scalar_t, typename scale_type, typename azp_type>
159 +__global__ void dynamic_scaled_int8_azp_quant_kernel(
160 + scalar_t const* __restrict__ input, int8_t* __restrict__ out,
161 + scale_type* scale, azp_type* azp, const int hidden_size) {
162 + int const token_idx = blockIdx.x;
163 +
164 + // Scan for the min and max value for this token
165 + float max_val = std::numeric_limits<float>::min();
166 + float min_val = std::numeric_limits<float>::max();
167 + for (int i = threadIdx.x; i < hidden_size; i += blockDim.x) {
168 + auto val = static_cast<float>(input[token_idx * hidden_size + i]);
169 + max_val = std::max(max_val, val);
170 + min_val = std::min(min_val, val);
171 + }
172 +
173 + // Reduce the max and min values across the block
174 + using BlockReduce = cub::BlockReduce<float, 1024>;
175 + __shared__ typename BlockReduce::TempStorage reduceStorage;
176 + max_val = BlockReduce(reduceStorage).Reduce(max_val, cub::Max{}, blockDim.x);
177 + __syncthreads(); // Make sure min doesn't mess with max shared memory
178 + min_val = BlockReduce(reduceStorage).Reduce(min_val, cub::Min{}, blockDim.x);
179 +
180 + __shared__ scale_type scale_sh;
181 + __shared__ azp_type azp_sh;
182 +
183 + // Compute the scale and zero point and store them, only on the first thread
184 + if (threadIdx.x == 0) {
185 + float const scale_val = (max_val - min_val) / 255.0f;
186 + // Use rounding to even (same as torch.round)
187 + auto const azp_float = std::nearbyint(-128.0f - min_val / scale_val);
188 + auto const azp_val = static_cast<azp_type>(azp_float);
189 +
190 + // Store the scale and azp into shared and global
191 + scale[token_idx] = scale_sh = scale_val;
192 + azp[token_idx] = azp_sh = azp_val;
193 + }
194 +
195 + // Wait for the scale and azp to be computed
196 + __syncthreads();
197 +
198 + float const scale_val = scale_sh;
199 + azp_type const azp_val = azp_sh;
200 +
201 + // Quantize the values
202 + for (int i = threadIdx.x; i < hidden_size; i += blockDim.x) {
203 + auto const val = static_cast<float>(input[token_idx * hidden_size + i]);
204 + auto const quant_val =
205 + int32_to_int8(float_to_int32_rn(val / scale_val) + azp_val);
206 + out[token_idx * hidden_size + i] = quant_val;
207 + }
208 +}
209 +
210 } // namespace vllm
211
212 void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
213 torch::Tensor const& input, // [..., hidden_size]
87 - torch::Tensor const& scale) {
214 + torch::Tensor const& scale,
215 + c10::optional<torch::Tensor> const& azp) {
216 TORCH_CHECK(input.is_contiguous());
217 TORCH_CHECK(out.is_contiguous());
218 TORCH_CHECK(scale.numel() == 1);
219 + TORCH_CHECK(!azp || azp->numel() == 1);
220
221 int const hidden_size = input.size(-1);
222 int const num_tokens = input.numel() / hidden_size;
@@ -96,19 +225,29 @@ void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
225 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
226 VLLM_DISPATCH_FLOATING_TYPES(
227 input.scalar_type(), "static_scaled_int8_quant_kernel", [&] {
99 - vllm::static_scaled_int8_quant_kernel<scalar_t, float>
100 - <<<grid, block, 0, stream>>>(input.data_ptr<scalar_t>(),
101 - out.data_ptr<int8_t>(),
102 - scale.data_ptr<float>(), hidden_size);
228 + if (!azp) {
229 + vllm::static_scaled_int8_quant_kernel<scalar_t, float>
230 + <<<grid, block, 0, stream>>>(
231 + input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
232 + scale.data_ptr<float>(), hidden_size);
233 + } else {
234 + vllm::static_scaled_int8_azp_quant_kernel<scalar_t, float, int32_t>
235 + <<<grid, block, 0, stream>>>(
236 + input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
237 + scale.data_ptr<float>(), azp->data_ptr<int32_t>(),
238 + hidden_size);
239 + }
240 });
241 }
242
243 void dynamic_scaled_int8_quant(
244 torch::Tensor& out, // [..., hidden_size]
245 torch::Tensor const& input, // [..., hidden_size]
109 - torch::Tensor& scales) {
246 + torch::Tensor& scales, c10::optional<torch::Tensor> const& azp) {
247 TORCH_CHECK(input.is_contiguous());
248 TORCH_CHECK(out.is_contiguous());
249 + TORCH_CHECK(scales.is_contiguous());
250 + TORCH_CHECK(!azp || azp->is_contiguous());
251
252 int const hidden_size = input.size(-1);
253 int const num_tokens = input.numel() / hidden_size;
@@ -117,9 +256,17 @@ void dynamic_scaled_int8_quant(
256 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
257 VLLM_DISPATCH_FLOATING_TYPES(
258 input.scalar_type(), "dynamic_scaled_int8_quant_kernel", [&] {
120 - vllm::dynamic_scaled_int8_quant_kernel<scalar_t, float>
121 - <<<grid, block, 0, stream>>>(input.data_ptr<scalar_t>(),
122 - out.data_ptr<int8_t>(),
123 - scales.data_ptr<float>(), hidden_size);
259 + if (!azp) {
260 + vllm::dynamic_scaled_int8_quant_kernel<scalar_t, float>
261 + <<<grid, block, 0, stream>>>(
262 + input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
263 + scales.data_ptr<float>(), hidden_size);
264 + } else {
265 + vllm::dynamic_scaled_int8_azp_quant_kernel<scalar_t, float, int32_t>
266 + <<<grid, block, 0, stream>>>(
267 + input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
268 + scales.data_ptr<float>(), azp->data_ptr<int32_t>(),
269 + hidden_size);
270 + }
271 });
272 }
csrc/torch_bindings.cpp
+4 -4
@@ -336,14 +336,14 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
336
337 // Compute int8 quantized tensor for given scaling factor.
338 ops.def(
339 - "static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale) -> "
340 - "()");
339 + "static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale,"
340 + "Tensor? azp) -> ()");
341 ops.impl("static_scaled_int8_quant", torch::kCUDA, &static_scaled_int8_quant);
342
343 // Compute int8 quantized tensor and scaling factor
344 ops.def(
345 - "dynamic_scaled_int8_quant(Tensor! out, Tensor input, Tensor! scale) -> "
346 - "()");
345 + "dynamic_scaled_int8_quant(Tensor! out, Tensor input, Tensor! scale, "
346 + "Tensor!? azp) -> ()");
347 ops.impl("dynamic_scaled_int8_quant", torch::kCUDA,
348 &dynamic_scaled_int8_quant);
349 }
tests/kernels/test_int8_quant.py
+138 -20
@@ -13,14 +13,28 @@ SEEDS = [0]
13 SCALE = [0.1, 0.5, 0.8, 1.2, 2.1]
14
15
16 -def opcheck_int8_quant(output, input, scale=None):
17 - if scale is not None:
18 - opcheck(torch.ops._C.static_scaled_int8_quant, (output, input, scale))
16 +def opcheck_int8_quant_static(output, input, scale, azp=None):
17 + if azp is None:
18 + opcheck(torch.ops._C.static_scaled_int8_quant,
19 + (output, input, scale, None))
20 else:
20 - scale = torch.empty((input.numel() // input.shape[-1], 1),
21 - device=input.device,
22 - dtype=torch.float32)
23 - opcheck(torch.ops._C.dynamic_scaled_int8_quant, (output, input, scale))
21 + opcheck(torch.ops._C.static_scaled_int8_quant,
22 + (output, input, scale, azp))
23 +
24 +
25 +def opcheck_int8_quant_dynamic(output, input, symmetric=True):
26 + scale = torch.empty((input.numel() // input.shape[-1], 1),
27 + device=input.device,
28 + dtype=torch.float32)
29 + if symmetric:
30 + opcheck(torch.ops._C.dynamic_scaled_int8_quant,
31 + (output, input, scale, None))
32 + else:
33 + azp = torch.empty((input.numel() // input.shape[-1], 1),
34 + device=input.device,
35 + dtype=torch.int32)
36 + opcheck(torch.ops._C.dynamic_scaled_int8_quant,
37 + (output, input, scale, azp))
38
39
40 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -38,14 +52,56 @@ def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
52 # reference
53 ref_out, ref_scales = ref_dynamic_per_token_quant(x, torch.int8)
54 # kernel
41 - ops_out, ops_scales = scaled_int8_quant(x)
55 + ops_out, ops_scales, _ = scaled_int8_quant(x)
56
57 torch.testing.assert_close(ops_scales, ref_scales)
44 - torch.testing.assert_close(
45 - ops_out, ref_out, atol=1,
46 - rtol=0.0) # big atol to account for rounding errors
58 + # big atol to account for rounding errors
59 + torch.testing.assert_close(ops_out, ref_out, atol=1, rtol=0.0)
60
48 - opcheck_int8_quant(ops_out, x)
61 + opcheck_int8_quant_dynamic(ops_out, x)
62 +
63 +
64 +@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
65 +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
66 +@pytest.mark.parametrize("dtype", DTYPES)
67 +@pytest.mark.parametrize("seed", SEEDS)
68 +@torch.inference_mode()
69 +def test_dynamic_scaled_int8_azp_quant(num_tokens: int, hidden_size: int,
70 + dtype: torch.dtype, seed: int) -> None:
71 + torch.random.manual_seed(seed)
72 + torch.cuda.manual_seed(seed)
73 + int8_traits = torch.iinfo(torch.int8)
74 +
75 + x = torch.rand(num_tokens, hidden_size, dtype=dtype,
76 + device="cuda") * 1000 - 300
77 +
78 + x_token_max, _ = x.to(dtype=torch.float32).max(dim=1, keepdim=True)
79 + x_token_min, _ = x.to(dtype=torch.float32).min(dim=1, keepdim=True)
80 +
81 + # calculate scale and azp, and adjust the range
82 + scales = (x_token_max - x_token_min) / torch.tensor(255.0)
83 + azps = torch.round(torch.tensor(-128.0) - x_token_min / scales).to(
84 + torch.int32)
85 +
86 + torch_out = ((x / scales).round() + azps).clamp(
87 + int8_traits.min, int8_traits.max).to(torch.int8)
88 + assert torch_out.min() >= int8_traits.min and torch_out.max(
89 + ) <= int8_traits.max
90 +
91 + ops_out = torch.empty_like(x, dtype=torch.int8)
92 + scales_out = torch.empty_like(scales, dtype=torch.float32)
93 + azp_out = torch.empty_like(azps, dtype=torch.int32)
94 + torch.ops._C.dynamic_scaled_int8_quant(ops_out, x, scales_out, azp_out)
95 +
96 + if (not torch.allclose(scales_out, scales)):
97 + print(torch.argmax(torch.abs(scales_out - scales)))
98 + torch.testing.assert_close(scales_out, scales)
99 + # big atol to account for rounding errors
100 + torch.testing.assert_close(azp_out, azps, atol=1, rtol=0.0)
101 + # if AZP is off by 1, after rounding-to-even, the output may be off by 2
102 + torch.testing.assert_close(ops_out, torch_out, atol=2, rtol=0.0)
103 +
104 + opcheck_int8_quant_dynamic(ops_out, x, False)
105
106
107 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -62,14 +118,76 @@ def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
118 int8_traits = torch.iinfo(torch.int8)
119
120 x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
65 - scale = torch.tensor([scale], dtype=torch.float32, device="cuda")
121 + scale_arg = torch.tensor([scale], dtype=torch.float32, device="cuda")
122 +
123 + out1 = (x / scale_arg).round().clamp(int8_traits.min,
124 + int8_traits.max).to(torch.int8)
125 + out2, _, _ = scaled_int8_quant(x, scale_arg)
126 +
127 + # big atol to account for rounding errors
128 + torch.testing.assert_close(out1, out2, atol=1, rtol=0.0)
129 +
130 + opcheck_int8_quant_static(out2, x, scale_arg)
131
67 - out1 = (x / scale).round().clamp(int8_traits.min,
68 - int8_traits.max).to(torch.int8)
69 - out2, _ = scaled_int8_quant(x, scale)
132
71 - torch.testing.assert_close(
72 - out1, out2, atol=1,
73 - rtol=0.0) # big atol to account for rounding errors
133 +@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
134 +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
135 +@pytest.mark.parametrize("dtype", DTYPES)
136 +@pytest.mark.parametrize("seed", SEEDS)
137 +@pytest.mark.parametrize("scale", SCALE[2:]) # Reduce test time
138 +@pytest.mark.parametrize("azp", [-255, 54])
139 +@torch.inference_mode()
140 +def test_static_scaled_int8_azp_quant(num_tokens: int, hidden_size: int,
141 + dtype: torch.dtype, seed: int,
142 + scale: float, azp: int) -> None:
143 + torch.random.manual_seed(seed)
144 + torch.cuda.manual_seed(seed)
145 + int8_traits = torch.iinfo(torch.int8)
146 +
147 + x = torch.rand(num_tokens, hidden_size, dtype=dtype,
148 + device="cuda") * 1000 - 300
149 +
150 + out1 = ((x / scale).round() + azp).clamp(int8_traits.min,
151 + int8_traits.max).to(torch.int8)
152 + out2 = torch.empty_like(x, dtype=torch.int8)
153 + scale_arg = torch.tensor([scale], dtype=torch.float32, device="cuda")
154 + azp_arg = torch.tensor([azp], dtype=torch.int32, device="cuda")
155 +
156 + torch.ops._C.static_scaled_int8_quant(out2, x, scale_arg, azp_arg)
157 +
158 + # big atol to account for rounding errors
159 + torch.testing.assert_close(out1, out2, atol=1, rtol=0.0)
160 +
161 + opcheck_int8_quant_static(out2, x, scale_arg, azp_arg)
162 +
163 +
164 +@pytest.mark.parametrize("is_max", [True, False])
165 +@torch.inference_mode()
166 +def test_static_scaled_int8_azp_quant_saturating_cast(is_max: bool) -> None:
167 + # Test that the saturating cast works correctly for values near i32 max/min
168 +
169 + from numpy import inf, nextafter
170 +
171 + int32_traits = torch.iinfo(torch.int32)
172 + val = float(int32_traits.max if is_max else int32_traits.min)
173 +
174 + x_vals = [[
175 + nextafter(val, inf), val + 1, val, val - 1,
176 + nextafter(val, -inf)
177 + ]]
178 + x = torch.tensor(x_vals, dtype=torch.float32, device="cuda")
179 +
180 + # The calculation in the kernel is: cast<int8>(cast<int32>(x / scale) + azp)
181 + # where cast<T> is a saturating cast to type T.
182 + # Scale is set to 1.0 so that the input values are the ones that are cast.
183 + # AZP is set to 0 to make sure the int8 saturating cast is tested as well.
184 + scale = torch.scalar_tensor(1.0, dtype=torch.float32, device="cuda")
185 + azp = torch.scalar_tensor(0, dtype=torch.int32, device="cuda")
186 +
187 + int8_traits = torch.iinfo(torch.int8)
188 + val_i8 = int8_traits.max if is_max else int8_traits.min
189 + expected = torch.full((1, 5), val_i8, dtype=torch.int8, device="cuda")
190
75 - opcheck_int8_quant(out2, x, scale)
191 + out = torch.empty_like(expected)
192 + torch.ops._C.static_scaled_int8_quant(out, x, scale, azp)
193 + torch.testing.assert_close(expected, out, atol=0, rtol=0)
vllm/_custom_ops.py
+20 -9
@@ -684,32 +684,43 @@ def scaled_fp8_quant(
684
685 # int8
686 def scaled_int8_quant(
687 - input: torch.Tensor,
688 - scale: Optional[torch.Tensor] = None
689 -) -> Tuple[torch.Tensor, torch.Tensor]:
687 + input: torch.Tensor,
688 + scale: Optional[torch.Tensor] = None,
689 + azp: Optional[torch.Tensor] = None,
690 + symmetric: bool = True
691 +) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
692 """
691 - Quantize the input tensor to int8 and return the quantized tensor and scale.
693 + Quantize the input tensor to int8 and return the quantized tensor and scale, and maybe azp.
694
695 Args:
696 input: The input tensor to be quantized to int8.
697 scale: Optional scaling factor for the int8 quantization.
698 When not provided, we invoke dynamic-per-token quantization.
699 + azp: Optional zero-point for the int8 quantization.
700 + Must be provided for asymmetric quantization if `scale` is provided.
701 + symmetric: Whether to use symmetric quantization (scale only, azp ignored).
702
703 Returns:
699 - Tuple[Torch.Tensor, Torch.Tensor] : Output int8 tensor and scales.
704 + Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]] : Output int8 tensor, scales, and optionally azp.
705 """
706 output = torch.empty_like(input, dtype=torch.int8)
707 if scale is not None:
708 # static-per-tensor quantization.
704 - torch.ops._C.static_scaled_int8_quant(output, input, scale)
705 - return output, scale
709 + assert symmetric == (
710 + azp is
711 + None), "azp must only be provided for asymmetric quantization."
712 + torch.ops._C.static_scaled_int8_quant(output, input, scale, azp)
713 + return output, scale, None
714
715 # dynamic-per-token quantization.
716 input_scales = torch.empty((input.numel() // input.shape[-1], 1),
717 device=input.device,
718 dtype=torch.float32)
711 - torch.ops._C.dynamic_scaled_int8_quant(output, input, input_scales)
712 - return output, input_scales
719 + input_azp = None if symmetric else torch.empty_like(input_scales,
720 + dtype=torch.int32)
721 + torch.ops._C.dynamic_scaled_int8_quant(output, input, input_scales,
722 + input_azp)
723 + return output, input_scales, input_azp
724
725
726 # qqq ops
vllm/model_executor/layers/quantization/qqq.py
+1 -1
@@ -260,7 +260,7 @@ class QQQLinearMethod(LinearMethodBase):
260 size_k = x_2d.shape[1]
261 size_n = s_ch.shape[1]
262
263 - x_int8, s_tok = ops.scaled_int8_quant(x_2d)
263 + x_int8, s_tok, _ = ops.scaled_int8_quant(x_2d)
264
265 output_2d = ops.marlin_qqq_gemm(x_int8, qweight, s_tok, s_ch, s_group,
266 workspace, size_m, size_n, size_k)
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+1 -1
@@ -188,7 +188,7 @@ def apply_int8_linear(
188 # ops.scaled_int8_quant supports both dynamic and static quant.
189 # * dynamic, layer.input_scale is None and x_scale computed from x.
190 # * static, layer.input_scale is scalar and x_scale is input_scale.
191 - x_q, x_scale = ops.scaled_int8_quant(input, input_scale)
191 + x_q, x_scale, _ = ops.scaled_int8_quant(input, input_scale)
192
193 return ops.cutlass_scaled_mm(x_q,
194 weight,