A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Kernel] AQ AZP 3/4: Asymmetric quantization kernels (#7270)
Luka Govedič committed
Sep 16, 2024 at 14:52 UTC
5d73ae49d65394f8dbe46accd921fb21e8247b82
9 files changed
+339
-57
csrc/cpu/quant.cpp
+6
-3
@@ -257,11 +257,13 @@ void int8_scaled_mm(torch::Tensor& c, // [M, OC], row-major
257
// static-per-tensor quantization.
258
void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
259
const torch::Tensor& input, // [..., hidden_size]
260
- const torch::Tensor& scale) {
260
+ const torch::Tensor& scale,
261
+ c10::optional<torch::Tensor> const& azp) {
262
CPU_KERNEL_GUARD_IN(static_scaled_int8_quant)
263
TORCH_CHECK(input.is_contiguous());
264
TORCH_CHECK(out.is_contiguous());
265
TORCH_CHECK(scale.numel() == 1);
266
+ TORCH_CHECK(!azp.has_value(), "Zero point is not supported on CPU.");
267
268
const int hidden_size = input.size(-1);
269
const int num_tokens = input.numel() / hidden_size;
@@ -277,11 +279,12 @@ void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
279
void dynamic_scaled_int8_quant(
280
torch::Tensor& out, // [..., hidden_size]
281
const torch::Tensor& input, // [..., hidden_size]
280
- torch::Tensor& scale // [..., 1]
281
-) {
282
+ torch::Tensor& scale, // [..., 1]
283
+ c10::optional<torch::Tensor> const& azp) {
284
CPU_KERNEL_GUARD_IN(dynamic_scaled_int8_quant)
285
TORCH_CHECK(input.is_contiguous());
286
TORCH_CHECK(out.is_contiguous());
287
+ TORCH_CHECK(!azp.has_value(), "Zero point is not supported on CPU.");
288
289
int const hidden_size = input.size(-1);
290
int const num_tokens = input.numel() / hidden_size;
csrc/cpu/torch_bindings.cpp
+5
-4
@@ -94,13 +94,14 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
94
#ifdef __AVX512F__
95
// Compute int8 quantized tensor for given scaling factor.
96
ops.def(
97
- "static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale) -> "
98
- "()");
97
+ "static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale,"
98
+ "Tensor? azp) -> ()");
99
ops.impl("static_scaled_int8_quant", torch::kCPU, &static_scaled_int8_quant);
100
+
101
// Compute int8 quantized tensor and scaling factor
102
ops.def(
102
- "dynamic_scaled_int8_quant(Tensor! out, Tensor input, Tensor! scale) -> "
103
- "()");
103
+ "dynamic_scaled_int8_quant(Tensor! out, Tensor input, Tensor! scale, "
104
+ "Tensor!? azp) -> ()");
105
ops.impl("dynamic_scaled_int8_quant", torch::kCPU,
106
&dynamic_scaled_int8_quant);
107
// W8A8 GEMM, supporting symmetric per-tensor or per-row/column
csrc/ops.h
+4
-2
@@ -184,10 +184,12 @@ torch::Tensor marlin_qqq_gemm(torch::Tensor const& a,
184
#endif
185
186
void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
187
- torch::Tensor const& scale);
187
+ torch::Tensor const& scale,
188
+ c10::optional<torch::Tensor> const& azp);
189
190
void dynamic_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
190
- torch::Tensor& scales);
191
+ torch::Tensor& scales,
192
+ c10::optional<torch::Tensor> const& azp);
193
194
torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
195
torch::Tensor b_gptq_qzeros,
csrc/quantization/compressed_tensors/int8_quant_kernels.cu
+160
-13
@@ -14,12 +14,17 @@
14
15
static inline __device__ int8_t float_to_int8_rn(float x) {
16
#ifdef USE_ROCM
17
- static const float i8_min =
17
+ static constexpr auto i8_min =
18
static_cast<float>(std::numeric_limits<int8_t>::min());
19
- static const float i8_max =
19
+ static constexpr auto i8_max =
20
static_cast<float>(std::numeric_limits<int8_t>::max());
21
- // round
21
+
22
+ // To match the rounding mode of CUDA, we use nearbyint.
23
+ // It uses the current rounding mode, which is always FE_TONEAREST on HIP.
24
+ // If that changes in the future, we may need to set the rounding mode
25
+ // explicitly, either at runtime or compile time.
26
float dst = std::nearbyint(x);
27
+
28
// saturate
29
dst = std::clamp(dst, i8_min, i8_max);
30
return static_cast<int8_t>(dst);
@@ -31,6 +36,59 @@ static inline __device__ int8_t float_to_int8_rn(float x) {
36
#endif
37
}
38
39
+static inline __device__ int32_t float_to_int32_rn(float x) {
40
+#ifdef USE_ROCM
41
+ // int32_max is not exactly representable as float.
42
+ // Therefore, we need to be careful and manually return int32_max on overflow.
43
+ // For symmetry, we also do the same for int32_min, even though it is exactly
44
+ // representable as float and the conversion should be exact.
45
+ static constexpr auto i32_min = std::numeric_limits<int32_t>::min();
46
+ static constexpr auto i32_min_f = static_cast<float>(i32_min);
47
+ static constexpr auto i32_max = std::numeric_limits<int32_t>::max();
48
+ static constexpr auto i32_max_f = static_cast<float>(i32_max);
49
+
50
+ // To match the rounding mode of CUDA, we use nearbyint.
51
+ // It uses the current rounding mode, which is always FE_TONEAREST on HIP.
52
+ // If that changes in the future, we may need to set the rounding mode
53
+ // explicitly, either at runtime or compile time.
54
+ float dst = std::nearbyint(x);
55
+
56
+ // saturate on the higher end.
57
+ if (dst >= i32_max_f) {
58
+ return i32_max;
59
+ }
60
+ // saturate on the lower end.
61
+ if (dst <= i32_min_f) {
62
+ return i32_min;
63
+ }
64
+
65
+ return static_cast<int32_t>(dst);
66
+#else
67
+ // CUDA path
68
+ uint32_t dst;
69
+ asm volatile("cvt.rni.sat.s32.f32 %0, %1;" : "=r"(dst) : "f"(x));
70
+ return reinterpret_cast<const int32_t&>(dst);
71
+#endif
72
+}
73
+
74
+static inline __device__ int8_t int32_to_int8(int32_t x) {
75
+#ifdef USE_ROCM
76
+ static constexpr auto i8_min =
77
+ static_cast<int32_t>(std::numeric_limits<int8_t>::min());
78
+ static constexpr auto i8_max =
79
+ static_cast<int32_t>(std::numeric_limits<int8_t>::max());
80
+
81
+ // saturate
82
+ int32_t dst = std::clamp(x, i8_min, i8_max);
83
+ return static_cast<int8_t>(dst);
84
+#else
85
+ // CUDA path
86
+ uint32_t dst;
87
+ asm volatile("cvt.sat.s8.s32 %0, %1;" : "=r"(dst) : "r"(x));
88
+ return reinterpret_cast<const int8_t&>(dst);
89
+#endif
90
+}
91
+
92
namespace vllm {
93
94
template <typename scalar_t, typename scale_type>
@@ -47,6 +105,23 @@ __global__ void static_scaled_int8_quant_kernel(
105
}
106
}
107
108
+template <typename scalar_t, typename scale_type, typename azp_type>
109
+__global__ void static_scaled_int8_azp_quant_kernel(
110
+ scalar_t const* __restrict__ input, int8_t* __restrict__ out,
111
+ scale_type const* scale_ptr, azp_type const* azp_ptr,
112
+ const int hidden_size) {
113
+ int const tid = threadIdx.x;
114
+ int const token_idx = blockIdx.x;
115
+ scale_type const scale = *scale_ptr;
116
+ azp_type const azp = *azp_ptr;
117
+
118
+ for (int i = tid; i < hidden_size; i += blockDim.x) {
119
+ auto const val = static_cast<float>(input[token_idx * hidden_size + i]);
120
+ auto const quant_val = int32_to_int8(float_to_int32_rn(val / scale) + azp);
121
+ out[token_idx * hidden_size + i] = quant_val;
122
+ }
123
+}
124
+
125
template <typename scalar_t, typename scale_type>
126
__global__ void dynamic_scaled_int8_quant_kernel(
127
scalar_t const* __restrict__ input, int8_t* __restrict__ out,
@@ -80,14 +155,68 @@ __global__ void dynamic_scaled_int8_quant_kernel(
155
}
156
}
157
158
+template <typename scalar_t, typename scale_type, typename azp_type>
159
+__global__ void dynamic_scaled_int8_azp_quant_kernel(
160
+ scalar_t const* __restrict__ input, int8_t* __restrict__ out,
161
+ scale_type* scale, azp_type* azp, const int hidden_size) {
162
+ int const token_idx = blockIdx.x;
163
+
164
+ // Scan for the min and max value for this token
165
+ float max_val = std::numeric_limits<float>::min();
166
+ float min_val = std::numeric_limits<float>::max();
167
+ for (int i = threadIdx.x; i < hidden_size; i += blockDim.x) {
168
+ auto val = static_cast<float>(input[token_idx * hidden_size + i]);
169
+ max_val = std::max(max_val, val);
170
+ min_val = std::min(min_val, val);
171
+ }
172
+
173
+ // Reduce the max and min values across the block
174
+ using BlockReduce = cub::BlockReduce<float, 1024>;
175
+ __shared__ typename BlockReduce::TempStorage reduceStorage;
176
+ max_val = BlockReduce(reduceStorage).Reduce(max_val, cub::Max{}, blockDim.x);
177
+ __syncthreads(); // Make sure min doesn't mess with max shared memory
178
+ min_val = BlockReduce(reduceStorage).Reduce(min_val, cub::Min{}, blockDim.x);
179
+
180
+ __shared__ scale_type scale_sh;
181
+ __shared__ azp_type azp_sh;
182
+
183
+ // Compute the scale and zero point and store them, only on the first thread
184
+ if (threadIdx.x == 0) {
185
+ float const scale_val = (max_val - min_val) / 255.0f;
186
+ // Use rounding to even (same as torch.round)
187
+ auto const azp_float = std::nearbyint(-128.0f - min_val / scale_val);
188
+ auto const azp_val = static_cast<azp_type>(azp_float);
189
+
190
+ // Store the scale and azp into shared and global
191
+ scale[token_idx] = scale_sh = scale_val;
192
+ azp[token_idx] = azp_sh = azp_val;
193
+ }
194
+
195
+ // Wait for the scale and azp to be computed
196
+ __syncthreads();
197
+
198
+ float const scale_val = scale_sh;
199
+ azp_type const azp_val = azp_sh;
200
+
201
+ // Quantize the values
202
+ for (int i = threadIdx.x; i < hidden_size; i += blockDim.x) {
203
+ auto const val = static_cast<float>(input[token_idx * hidden_size + i]);
204
+ auto const quant_val =
205
+ int32_to_int8(float_to_int32_rn(val / scale_val) + azp_val);
206
+ out[token_idx * hidden_size + i] = quant_val;
207
+ }
208
+}
209
+
210
} // namespace vllm
211
212
void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
213
torch::Tensor const& input, // [..., hidden_size]
87
- torch::Tensor const& scale) {
214
+ torch::Tensor const& scale,
215
+ c10::optional<torch::Tensor> const& azp) {
216
TORCH_CHECK(input.is_contiguous());
217
TORCH_CHECK(out.is_contiguous());
218
TORCH_CHECK(scale.numel() == 1);
219
+ TORCH_CHECK(!azp || azp->numel() == 1);
220
221
int const hidden_size = input.size(-1);
222
int const num_tokens = input.numel() / hidden_size;
@@ -96,19 +225,29 @@ void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
225
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
226
VLLM_DISPATCH_FLOATING_TYPES(
227
input.scalar_type(), "static_scaled_int8_quant_kernel", [&] {
99
- vllm::static_scaled_int8_quant_kernel<scalar_t, float>
100
- <<<grid, block, 0, stream>>>(input.data_ptr<scalar_t>(),
101
- out.data_ptr<int8_t>(),
102
- scale.data_ptr<float>(), hidden_size);
228
+ if (!azp) {
229
+ vllm::static_scaled_int8_quant_kernel<scalar_t, float>
230
+ <<<grid, block, 0, stream>>>(
231
+ input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
232
+ scale.data_ptr<float>(), hidden_size);
233
+ } else {
234
+ vllm::static_scaled_int8_azp_quant_kernel<scalar_t, float, int32_t>
235
+ <<<grid, block, 0, stream>>>(
236
+ input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
237
+ scale.data_ptr<float>(), azp->data_ptr<int32_t>(),
238
+ hidden_size);
239
+ }
240
});
241
}
242
243
void dynamic_scaled_int8_quant(
244
torch::Tensor& out, // [..., hidden_size]
245
torch::Tensor const& input, // [..., hidden_size]
109
- torch::Tensor& scales) {
246
+ torch::Tensor& scales, c10::optional<torch::Tensor> const& azp) {
247
TORCH_CHECK(input.is_contiguous());
248
TORCH_CHECK(out.is_contiguous());
249
+ TORCH_CHECK(scales.is_contiguous());
250
+ TORCH_CHECK(!azp || azp->is_contiguous());
251
252
int const hidden_size = input.size(-1);
253
int const num_tokens = input.numel() / hidden_size;
@@ -117,9 +256,17 @@ void dynamic_scaled_int8_quant(
256
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
257
VLLM_DISPATCH_FLOATING_TYPES(
258
input.scalar_type(), "dynamic_scaled_int8_quant_kernel", [&] {
120
- vllm::dynamic_scaled_int8_quant_kernel<scalar_t, float>
121
- <<<grid, block, 0, stream>>>(input.data_ptr<scalar_t>(),
122
- out.data_ptr<int8_t>(),
123
- scales.data_ptr<float>(), hidden_size);
259
+ if (!azp) {
260
+ vllm::dynamic_scaled_int8_quant_kernel<scalar_t, float>
261
+ <<<grid, block, 0, stream>>>(
262
+ input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
263
+ scales.data_ptr<float>(), hidden_size);
264
+ } else {
265
+ vllm::dynamic_scaled_int8_azp_quant_kernel<scalar_t, float, int32_t>
266
+ <<<grid, block, 0, stream>>>(
267
+ input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
268
+ scales.data_ptr<float>(), azp->data_ptr<int32_t>(),
269
+ hidden_size);
270
+ }
271
});
272
}
csrc/torch_bindings.cpp
+4
-4
@@ -336,14 +336,14 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
336
337
// Compute int8 quantized tensor for given scaling factor.
338
ops.def(
339
- "static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale) -> "
340
- "()");
339
+ "static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale,"
340
+ "Tensor? azp) -> ()");
341
ops.impl("static_scaled_int8_quant", torch::kCUDA, &static_scaled_int8_quant);
342
343
// Compute int8 quantized tensor and scaling factor
344
ops.def(
345
- "dynamic_scaled_int8_quant(Tensor! out, Tensor input, Tensor! scale) -> "
346
- "()");
345
+ "dynamic_scaled_int8_quant(Tensor! out, Tensor input, Tensor! scale, "
346
+ "Tensor!? azp) -> ()");
347
ops.impl("dynamic_scaled_int8_quant", torch::kCUDA,
348
&dynamic_scaled_int8_quant);
349
}
tests/kernels/test_int8_quant.py
+138
-20
@@ -13,14 +13,28 @@ SEEDS = [0]
13
SCALE = [0.1, 0.5, 0.8, 1.2, 2.1]
14
15
16
-def opcheck_int8_quant(output, input, scale=None):
17
- if scale is not None:
18
- opcheck(torch.ops._C.static_scaled_int8_quant, (output, input, scale))
16
+def opcheck_int8_quant_static(output, input, scale, azp=None):
17
+ if azp is None:
18
+ opcheck(torch.ops._C.static_scaled_int8_quant,
19
+ (output, input, scale, None))
20
else:
20
- scale = torch.empty((input.numel() // input.shape[-1], 1),
21
- device=input.device,
22
- dtype=torch.float32)
23
- opcheck(torch.ops._C.dynamic_scaled_int8_quant, (output, input, scale))
21
+ opcheck(torch.ops._C.static_scaled_int8_quant,
22
+ (output, input, scale, azp))
23
+
24
+
25
+def opcheck_int8_quant_dynamic(output, input, symmetric=True):
26
+ scale = torch.empty((input.numel() // input.shape[-1], 1),
27
+ device=input.device,
28
+ dtype=torch.float32)
29
+ if symmetric:
30
+ opcheck(torch.ops._C.dynamic_scaled_int8_quant,
31
+ (output, input, scale, None))
32
+ else:
33
+ azp = torch.empty((input.numel() // input.shape[-1], 1),
34
+ device=input.device,
35
+ dtype=torch.int32)
36
+ opcheck(torch.ops._C.dynamic_scaled_int8_quant,
37
+ (output, input, scale, azp))
38
39
40
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -38,14 +52,56 @@ def test_dynamic_scaled_int8_quant(num_tokens: int, hidden_size: int,
52
# reference
53
ref_out, ref_scales = ref_dynamic_per_token_quant(x, torch.int8)
54
# kernel
41
- ops_out, ops_scales = scaled_int8_quant(x)
55
+ ops_out, ops_scales, _ = scaled_int8_quant(x)
56
57
torch.testing.assert_close(ops_scales, ref_scales)
44
- torch.testing.assert_close(
45
- ops_out, ref_out, atol=1,
46
- rtol=0.0) # big atol to account for rounding errors
58
+ # big atol to account for rounding errors
59
+ torch.testing.assert_close(ops_out, ref_out, atol=1, rtol=0.0)
60
48
- opcheck_int8_quant(ops_out, x)
61
+ opcheck_int8_quant_dynamic(ops_out, x)
62
+
63
+
64
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
65
+@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
66
+@pytest.mark.parametrize("dtype", DTYPES)
67
+@pytest.mark.parametrize("seed", SEEDS)
68
+@torch.inference_mode()
69
+def test_dynamic_scaled_int8_azp_quant(num_tokens: int, hidden_size: int,
70
+ dtype: torch.dtype, seed: int) -> None:
71
+ torch.random.manual_seed(seed)
72
+ torch.cuda.manual_seed(seed)
73
+ int8_traits = torch.iinfo(torch.int8)
74
+
75
+ x = torch.rand(num_tokens, hidden_size, dtype=dtype,
76
+ device="cuda") * 1000 - 300
77
+
78
+ x_token_max, _ = x.to(dtype=torch.float32).max(dim=1, keepdim=True)
79
+ x_token_min, _ = x.to(dtype=torch.float32).min(dim=1, keepdim=True)
80
+
81
+ # calculate scale and azp, and adjust the range
82
+ scales = (x_token_max - x_token_min) / torch.tensor(255.0)
83
+ azps = torch.round(torch.tensor(-128.0) - x_token_min / scales).to(
84
+ torch.int32)
85
+
86
+ torch_out = ((x / scales).round() + azps).clamp(
87
+ int8_traits.min, int8_traits.max).to(torch.int8)
88
+ assert torch_out.min() >= int8_traits.min and torch_out.max(
89
+ ) <= int8_traits.max
90
+
91
+ ops_out = torch.empty_like(x, dtype=torch.int8)
92
+ scales_out = torch.empty_like(scales, dtype=torch.float32)
93
+ azp_out = torch.empty_like(azps, dtype=torch.int32)
94
+ torch.ops._C.dynamic_scaled_int8_quant(ops_out, x, scales_out, azp_out)
95
+
96
+ if (not torch.allclose(scales_out, scales)):
97
+ print(torch.argmax(torch.abs(scales_out - scales)))
98
+ torch.testing.assert_close(scales_out, scales)
99
+ # big atol to account for rounding errors
100
+ torch.testing.assert_close(azp_out, azps, atol=1, rtol=0.0)
101
+ # if AZP is off by 1, after rounding-to-even, the output may be off by 2
102
+ torch.testing.assert_close(ops_out, torch_out, atol=2, rtol=0.0)
103
+
104
+ opcheck_int8_quant_dynamic(ops_out, x, False)
105
106
107
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -62,14 +118,76 @@ def test_static_scaled_int8_quant(num_tokens: int, hidden_size: int,
118
int8_traits = torch.iinfo(torch.int8)
119
120
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") * 1000
65
- scale = torch.tensor([scale], dtype=torch.float32, device="cuda")
121
+ scale_arg = torch.tensor([scale], dtype=torch.float32, device="cuda")
122
+
123
+ out1 = (x / scale_arg).round().clamp(int8_traits.min,
124
+ int8_traits.max).to(torch.int8)
125
+ out2, _, _ = scaled_int8_quant(x, scale_arg)
126
+
127
+ # big atol to account for rounding errors
128
+ torch.testing.assert_close(out1, out2, atol=1, rtol=0.0)
129
+
130
+ opcheck_int8_quant_static(out2, x, scale_arg)
131
67
- out1 = (x / scale).round().clamp(int8_traits.min,
68
- int8_traits.max).to(torch.int8)
69
- out2, _ = scaled_int8_quant(x, scale)
132
71
- torch.testing.assert_close(
72
- out1, out2, atol=1,
73
- rtol=0.0) # big atol to account for rounding errors
133
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
134
+@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
135
+@pytest.mark.parametrize("dtype", DTYPES)
136
+@pytest.mark.parametrize("seed", SEEDS)
137
+@pytest.mark.parametrize("scale", SCALE[2:]) # Reduce test time
138
+@pytest.mark.parametrize("azp", [-255, 54])
139
+@torch.inference_mode()
140
+def test_static_scaled_int8_azp_quant(num_tokens: int, hidden_size: int,
141
+ dtype: torch.dtype, seed: int,
142
+ scale: float, azp: int) -> None:
143
+ torch.random.manual_seed(seed)
144
+ torch.cuda.manual_seed(seed)
145
+ int8_traits = torch.iinfo(torch.int8)
146
+
147
+ x = torch.rand(num_tokens, hidden_size, dtype=dtype,
148
+ device="cuda") * 1000 - 300
149
+
150
+ out1 = ((x / scale).round() + azp).clamp(int8_traits.min,
151
+ int8_traits.max).to(torch.int8)
152
+ out2 = torch.empty_like(x, dtype=torch.int8)
153
+ scale_arg = torch.tensor([scale], dtype=torch.float32, device="cuda")
154
+ azp_arg = torch.tensor([azp], dtype=torch.int32, device="cuda")
155
+
156
+ torch.ops._C.static_scaled_int8_quant(out2, x, scale_arg, azp_arg)
157
+
158
+ # big atol to account for rounding errors
159
+ torch.testing.assert_close(out1, out2, atol=1, rtol=0.0)
160
+
161
+ opcheck_int8_quant_static(out2, x, scale_arg, azp_arg)
162
+
163
+
164
+@pytest.mark.parametrize("is_max", [True, False])
165
+@torch.inference_mode()
166
+def test_static_scaled_int8_azp_quant_saturating_cast(is_max: bool) -> None:
167
+ # Test that the saturating cast works correctly for values near i32 max/min
168
+
169
+ from numpy import inf, nextafter
170
+
171
+ int32_traits = torch.iinfo(torch.int32)
172
+ val = float(int32_traits.max if is_max else int32_traits.min)
173
+
174
+ x_vals = [[
175
+ nextafter(val, inf), val + 1, val, val - 1,
176
+ nextafter(val, -inf)
177
+ ]]
178
+ x = torch.tensor(x_vals, dtype=torch.float32, device="cuda")
179
+
180
+ # The calculation in the kernel is: cast<int8>(cast<int32>(x / scale) + azp)
181
+ # where cast<T> is a saturating cast to type T.
182
+ # Scale is set to 1.0 so that the input values are the ones that are cast.
183
+ # AZP is set to 0 to make sure the int8 saturating cast is tested as well.
184
+ scale = torch.scalar_tensor(1.0, dtype=torch.float32, device="cuda")
185
+ azp = torch.scalar_tensor(0, dtype=torch.int32, device="cuda")
186
+
187
+ int8_traits = torch.iinfo(torch.int8)
188
+ val_i8 = int8_traits.max if is_max else int8_traits.min
189
+ expected = torch.full((1, 5), val_i8, dtype=torch.int8, device="cuda")
190
75
- opcheck_int8_quant(out2, x, scale)
191
+ out = torch.empty_like(expected)
192
+ torch.ops._C.static_scaled_int8_quant(out, x, scale, azp)
193
+ torch.testing.assert_close(expected, out, atol=0, rtol=0)
vllm/_custom_ops.py
+20
-9
@@ -684,32 +684,43 @@ def scaled_fp8_quant(
684
685
# int8
686
def scaled_int8_quant(
687
- input: torch.Tensor,
688
- scale: Optional[torch.Tensor] = None
689
-) -> Tuple[torch.Tensor, torch.Tensor]:
687
+ input: torch.Tensor,
688
+ scale: Optional[torch.Tensor] = None,
689
+ azp: Optional[torch.Tensor] = None,
690
+ symmetric: bool = True
691
+) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
692
"""
691
- Quantize the input tensor to int8 and return the quantized tensor and scale.
693
+ Quantize the input tensor to int8 and return the quantized tensor and scale, and maybe azp.
694
695
Args:
696
input: The input tensor to be quantized to int8.
697
scale: Optional scaling factor for the int8 quantization.
698
When not provided, we invoke dynamic-per-token quantization.
699
+ azp: Optional zero-point for the int8 quantization.
700
+ Must be provided for asymmetric quantization if `scale` is provided.
701
+ symmetric: Whether to use symmetric quantization (scale only, azp ignored).
702
703
Returns:
699
- Tuple[Torch.Tensor, Torch.Tensor] : Output int8 tensor and scales.
704
+ Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]] : Output int8 tensor, scales, and optionally azp.
705
"""
706
output = torch.empty_like(input, dtype=torch.int8)
707
if scale is not None:
708
# static-per-tensor quantization.
704
- torch.ops._C.static_scaled_int8_quant(output, input, scale)
705
- return output, scale
709
+ assert symmetric == (
710
+ azp is
711
+ None), "azp must only be provided for asymmetric quantization."
712
+ torch.ops._C.static_scaled_int8_quant(output, input, scale, azp)
713
+ return output, scale, None
714
715
# dynamic-per-token quantization.
716
input_scales = torch.empty((input.numel() // input.shape[-1], 1),
717
device=input.device,
718
dtype=torch.float32)
711
- torch.ops._C.dynamic_scaled_int8_quant(output, input, input_scales)
712
- return output, input_scales
719
+ input_azp = None if symmetric else torch.empty_like(input_scales,
720
+ dtype=torch.int32)
721
+ torch.ops._C.dynamic_scaled_int8_quant(output, input, input_scales,
722
+ input_azp)
723
+ return output, input_scales, input_azp
724
725
726
# qqq ops
vllm/model_executor/layers/quantization/qqq.py
+1
-1
@@ -260,7 +260,7 @@ class QQQLinearMethod(LinearMethodBase):
260
size_k = x_2d.shape[1]
261
size_n = s_ch.shape[1]
262
263
- x_int8, s_tok = ops.scaled_int8_quant(x_2d)
263
+ x_int8, s_tok, _ = ops.scaled_int8_quant(x_2d)
264
265
output_2d = ops.marlin_qqq_gemm(x_int8, qweight, s_tok, s_ch, s_group,
266
workspace, size_m, size_n, size_k)
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+1
-1
@@ -188,7 +188,7 @@ def apply_int8_linear(
188
# ops.scaled_int8_quant supports both dynamic and static quant.
189
# * dynamic, layer.input_scale is None and x_scale computed from x.
190
# * static, layer.input_scale is scalar and x_scale is input_scale.
191
- x_q, x_scale = ops.scaled_int8_quant(input, input_scale)
191
+ x_q, x_scale, _ = ops.scaled_int8_quant(input, input_scale)
192
193
return ops.cutlass_scaled_mm(x_q,
194
weight,