A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Hardware][CPU] compressed-tensor INT8 W8A8 AZP support (#9344)
Li, Jiang committed
Oct 18, 2024 at 00:21 UTC
5eda21e773447d81ffc661ac094716420dc7b7cb
7 files changed
+452
-96
.buildkite/run-cpu-test.sh
+4
-4
@@ -32,10 +32,10 @@ docker exec cpu-test bash -c "
32
--ignore=tests/models/decoder_only/language/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
33
34
# Run compressed-tensor test
35
-# docker exec cpu-test bash -c "
36
-# pytest -s -v \
37
-# tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_static_setup \
38
-# tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_dynanmic_per_token"
35
+docker exec cpu-test bash -c "
36
+ pytest -s -v \
37
+ tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_static_setup \
38
+ tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_dynamic_per_token"
39
40
# Run AWQ test
41
docker exec cpu-test bash -c "
Dockerfile.cpu
-13
@@ -33,19 +33,6 @@ RUN --mount=type=cache,target=/root/.cache/pip \
33
pip install --upgrade pip && \
34
pip install -r requirements-build.txt
35
36
-# install oneDNN
37
-RUN git clone -b rls-v3.5 https://github.com/oneapi-src/oneDNN.git
38
-
39
-RUN --mount=type=cache,target=/root/.cache/ccache \
40
- cmake -B ./oneDNN/build -S ./oneDNN -G Ninja -DONEDNN_LIBRARY_TYPE=STATIC \
41
- -DONEDNN_BUILD_DOC=OFF \
42
- -DONEDNN_BUILD_EXAMPLES=OFF \
43
- -DONEDNN_BUILD_TESTS=OFF \
44
- -DONEDNN_BUILD_GRAPH=OFF \
45
- -DONEDNN_ENABLE_WORKLOAD=INFERENCE \
46
- -DONEDNN_ENABLE_PRIMITIVE=MATMUL && \
47
- cmake --build ./oneDNN/build --target install --config Release
48
-
36
FROM cpu-test-1 AS build
37
38
WORKDIR /workspace/vllm
cmake/cpu_extension.cmake
+34
-6
@@ -1,5 +1,8 @@
1
+include(FetchContent)
2
+
3
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
4
+set(CMAKE_CXX_EXTENSIONS ON)
5
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
2
-set(CMAKE_CXX_STANDARD 17)
6
7
#
8
# Define environment variables for special configurations
@@ -82,15 +85,40 @@ else()
85
message(FATAL_ERROR "vLLM CPU backend requires AVX512 or AVX2 or Power9+ ISA support.")
86
endif()
87
88
+#
89
+# Build oneDNN for W8A8 GEMM kernels (only for x86-AVX512 platforms)
90
+#
91
+if (AVX512_FOUND AND NOT AVX512_DISABLED)
92
+ FetchContent_Declare(
93
+ oneDNN
94
+ GIT_REPOSITORY https://github.com/oneapi-src/oneDNN.git
95
+ GIT_TAG v3.5.3
96
+ GIT_PROGRESS TRUE
97
+ GIT_SHALLOW TRUE
98
+ )
99
+
100
+ set(ONEDNN_LIBRARY_TYPE "STATIC")
101
+ set(ONEDNN_BUILD_DOC "OFF")
102
+ set(ONEDNN_BUILD_EXAMPLES "OFF")
103
+ set(ONEDNN_BUILD_TESTS "OFF")
104
+ set(ONEDNN_ENABLE_WORKLOAD "INFERENCE")
105
+ set(ONEDNN_ENABLE_PRIMITIVE "MATMUL;REORDER")
106
+ set(ONEDNN_BUILD_GRAPH "OFF")
107
+ set(ONEDNN_ENABLE_JIT_PROFILING "OFF")
108
+ set(ONEDNN_ENABLE_ITT_TASKS "OFF")
109
+ set(ONEDNN_ENABLE_MAX_CPU_ISA "OFF")
110
+ set(ONEDNN_ENABLE_CPU_ISA_HINTS "OFF")
111
+ set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
112
+
113
+ FetchContent_MakeAvailable(oneDNN)
114
+
115
+ list(APPEND LIBS dnnl)
116
+endif()
117
+
118
message(STATUS "CPU extension compile flags: ${CXX_COMPILE_FLAGS}")
119
120
list(APPEND LIBS numa)
121
89
-# Appending the dnnl library for the AVX2 and AVX512, as it is not utilized by Power architecture.
90
-if (AVX2_FOUND OR AVX512_FOUND)
91
- list(APPEND LIBS dnnl)
92
-endif()
93
-
122
#
123
# _C extension
124
#
csrc/cpu/cpu_types_x86.hpp
+39
-2
@@ -265,6 +265,30 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
265
void save(float *ptr) const { _mm256_storeu_ps(ptr, reg); }
266
};
267
268
+#ifdef __AVX512F__
269
+struct INT32Vec16: public Vec<INT32Vec16> {
270
+ constexpr static int VEC_ELEM_NUM = 16;
271
+ union AliasReg {
272
+ __m512i reg;
273
+ int32_t values[VEC_ELEM_NUM];
274
+ };
275
+
276
+ __m512i reg;
277
+
278
+ explicit INT32Vec16(const void* data_ptr) : reg(_mm512_loadu_epi32(data_ptr)) {}
279
+
280
+ void save(int32_t* ptr) const {
281
+ _mm512_storeu_epi32(ptr, reg);
282
+ }
283
+
284
+ void save(int32_t* ptr, const int elem_num) const {
285
+ constexpr uint32_t M = 0xFFFFFFFF;
286
+ __mmask16 mask = _cvtu32_mask16(M >> (32 - elem_num));
287
+ _mm512_mask_storeu_epi32(ptr, mask, reg);
288
+ }
289
+};
290
+#endif
291
+
292
#ifdef __AVX512F__
293
struct FP32Vec16 : public Vec<FP32Vec16> {
294
constexpr static int VEC_ELEM_NUM = 16;
@@ -283,8 +307,6 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
307
308
explicit FP32Vec16(__m512 data) : reg(data) {}
309
286
- explicit FP32Vec16(const FP32Vec16 &data) : reg(data.reg) {}
287
-
310
explicit FP32Vec16(const FP32Vec4 &data)
311
: reg((__m512)_mm512_inserti32x4(
312
_mm512_inserti32x4(
@@ -303,6 +325,9 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
325
326
explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
327
328
+ explicit FP32Vec16(const INT32Vec16 &v)
329
+ : reg(_mm512_cvt_roundepi32_ps(v.reg, _MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC)) {}
330
+
331
FP32Vec16 operator*(const FP32Vec16 &b) const {
332
return FP32Vec16(_mm512_mul_ps(reg, b.reg));
333
}
@@ -333,6 +358,16 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
358
return FP32Vec16(_mm512_mask_max_ps(reg, mask, reg, b.reg));
359
}
360
361
+ FP32Vec16 min(const FP32Vec16& b) const {
362
+ return FP32Vec16(_mm512_min_ps(reg, b.reg));
363
+ }
364
+
365
+ FP32Vec16 min(const FP32Vec16& b, const int elem_num) const {
366
+ constexpr uint32_t M = 0xFFFFFFFF;
367
+ __mmask16 mask = _cvtu32_mask16(M >> (32 - elem_num));
368
+ return FP32Vec16(_mm512_mask_min_ps(reg, mask, reg, b.reg));
369
+ }
370
+
371
FP32Vec16 abs() const {
372
return FP32Vec16(_mm512_abs_ps(reg));
373
}
@@ -341,6 +376,8 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
376
377
float reduce_max() const { return _mm512_reduce_max_ps(reg); }
378
379
+ float reduce_min() const { return _mm512_reduce_min_ps(reg); }
380
+
381
template <int group_size> float reduce_sub_sum(int idx) {
382
static_assert(VEC_ELEM_NUM % group_size == 0);
383
constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
csrc/cpu/quant.cpp
+360
-57
@@ -5,25 +5,29 @@ namespace {
5
template <typename scalar_t>
6
struct KernelVecType {
7
using load_vec_type = void;
8
+ using azp_adj_load_vec_type = void;
9
using cvt_vec_type = void;
10
};
11
12
template <>
13
struct KernelVecType<float> {
14
using load_vec_type = vec_op::FP32Vec16;
15
+ using azp_adj_load_vec_type = vec_op::INT32Vec16;
16
using cvt_vec_type = vec_op::FP32Vec16;
17
};
18
19
template <>
20
struct KernelVecType<c10::BFloat16> {
21
using load_vec_type = vec_op::BF16Vec16;
22
+ using azp_adj_load_vec_type = vec_op::INT32Vec16;
23
using cvt_vec_type = vec_op::FP32Vec16;
24
};
25
26
#ifdef __AVX512F__
24
-template <typename scalar_t>
27
+template <bool AZP, typename scalar_t>
28
void static_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
26
- const float* scale, const int num_tokens,
29
+ const float* scale, const int32_t* azp,
30
+ const int num_tokens,
31
const int hidden_size) {
32
using load_vec_t = typename KernelVecType<scalar_t>::load_vec_type;
33
using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
@@ -37,62 +41,110 @@ void static_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
41
const cvt_vec_t i8_min_vec(i8_min);
42
const cvt_vec_t i8_max_vec(i8_max);
43
44
+ cvt_vec_t zp_vec;
45
+ if constexpr (AZP) {
46
+ zp_vec = cvt_vec_t(static_cast<float>(*azp));
47
+ }
48
+
49
#pragma omp parallel for
50
for (int i = 0; i < num_tokens; ++i) {
51
int j = 0;
52
for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
53
load_vec_t elems(input + i * hidden_size + j);
54
cvt_vec_t elems_fp32(elems);
46
- elems_fp32 = (elems_fp32 * inv_scale).clamp(i8_min_vec, i8_max_vec);
55
+ elems_fp32 = elems_fp32 * inv_scale;
56
+
57
+ if constexpr (AZP) {
58
+ elems_fp32 = elems_fp32 + zp_vec;
59
+ }
60
+
61
+ elems_fp32 = elems_fp32.clamp(i8_min_vec, i8_max_vec);
62
vec_op::INT8Vec16 elems_int8(elems_fp32);
63
elems_int8.save(output + i * hidden_size + j);
64
}
65
66
load_vec_t elems(input + i * hidden_size + j);
67
cvt_vec_t elems_fp32(elems);
53
- elems_fp32 = (elems_fp32 * inv_scale).clamp(i8_min_vec, i8_max_vec);
54
- vec_op::INT8Vec16 elems_int8(elems_fp32);
68
+ elems_fp32 = elems_fp32 * inv_scale;
69
56
- if (j + vec_elem_num == hidden_size) {
57
- elems_int8.save(output + i * hidden_size + j);
58
- } else {
59
- elems_int8.save(output + i * hidden_size + j, hidden_size - j);
70
+ if constexpr (AZP) {
71
+ elems_fp32 = elems_fp32 + zp_vec;
72
}
73
+
74
+ elems_fp32 = elems_fp32.clamp(i8_min_vec, i8_max_vec);
75
+ vec_op::INT8Vec16 elems_int8(elems_fp32);
76
+ elems_int8.save(output + i * hidden_size + j, hidden_size - j);
77
}
78
}
79
64
-template <typename scalar_t>
80
+template <bool AZP, typename scalar_t>
81
void dynamic_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
66
- float* scale, const int num_tokens,
82
+ float* scale, int32_t* azp,
83
+ const int num_tokens,
84
const int hidden_size) {
85
using load_vec_t = typename KernelVecType<scalar_t>::load_vec_type;
86
using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
87
constexpr int vec_elem_num = load_vec_t::VEC_ELEM_NUM;
88
89
+ constexpr float i8_min =
90
+ static_cast<float>(std::numeric_limits<int8_t>::min());
91
+ constexpr float i8_max =
92
+ static_cast<float>(std::numeric_limits<int8_t>::max());
93
+ const cvt_vec_t i8_min_vec(i8_min);
94
+ const cvt_vec_t i8_max_vec(i8_max);
95
+
96
#pragma omp parallel for
97
for (int i = 0; i < num_tokens; ++i) {
74
- cvt_vec_t max_abs(0.0);
98
+ cvt_vec_t max_value(std::numeric_limits<float>::lowest());
99
+ cvt_vec_t min_value(std::numeric_limits<float>::max());
100
{
101
int j = 0;
102
for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
103
load_vec_t elems(input + i * hidden_size + j);
104
cvt_vec_t elems_fp32(elems);
80
- max_abs = max_abs.max(elems_fp32.abs());
105
+ if constexpr (AZP) {
106
+ max_value = max_value.max(elems_fp32);
107
+ min_value = min_value.min(elems_fp32);
108
+ } else {
109
+ max_value = max_value.max(elems_fp32.abs());
110
+ }
111
}
112
113
load_vec_t elems(input + i * hidden_size + j);
114
cvt_vec_t elems_fp32(elems);
115
116
if (j + vec_elem_num == hidden_size) {
87
- max_abs = max_abs.max(elems_fp32.abs());
117
+ if constexpr (AZP) {
118
+ max_value = max_value.max(elems_fp32);
119
+ min_value = min_value.min(elems_fp32);
120
+ } else {
121
+ max_value = max_value.max(elems_fp32.abs());
122
+ }
123
} else {
89
- max_abs = max_abs.max(elems_fp32.abs(), hidden_size - j);
124
+ if constexpr (AZP) {
125
+ max_value = max_value.max(elems_fp32, hidden_size - j);
126
+ min_value = min_value.min(elems_fp32, hidden_size - j);
127
+ } else {
128
+ max_value = max_value.max(elems_fp32.abs(), hidden_size - j);
129
+ }
130
}
131
}
132
93
- float scale_val = max_abs.reduce_max() / 127.0f;
94
- scale[i] = scale_val;
133
+ float scale_val, azp_val;
134
+ if constexpr (AZP) {
135
+ float max_scalar = max_value.reduce_max();
136
+ float min_scalar = min_value.reduce_min();
137
+ scale_val = (max_scalar - min_scalar) / 255.0f;
138
+ azp_val = std::nearbyint(-128.0f - min_scalar / scale_val);
139
+ azp[i] = static_cast<int32_t>(azp_val);
140
+ scale[i] = scale_val;
141
+ } else {
142
+ scale_val = max_value.reduce_max() / 127.0f;
143
+ scale[i] = scale_val;
144
+ }
145
+
146
const cvt_vec_t inv_scale(1.0 / scale_val);
147
+ const cvt_vec_t azp_vec(azp_val);
148
149
{
150
int j = 0;
@@ -100,6 +152,11 @@ void dynamic_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
152
load_vec_t elems(input + i * hidden_size + j);
153
cvt_vec_t elems_fp32(elems);
154
elems_fp32 = (elems_fp32 * inv_scale);
155
+
156
+ if constexpr (AZP) {
157
+ elems_fp32 = elems_fp32 + azp_vec;
158
+ }
159
+ elems_fp32 = elems_fp32.clamp(i8_min_vec, i8_max_vec);
160
vec_op::INT8Vec16 elems_int8(elems_fp32);
161
elems_int8.save(output + i * hidden_size + j);
162
}
@@ -107,34 +164,111 @@ void dynamic_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
164
load_vec_t elems(input + i * hidden_size + j);
165
cvt_vec_t elems_fp32(elems);
166
elems_fp32 = (elems_fp32 * inv_scale);
110
- vec_op::INT8Vec16 elems_int8(elems_fp32);
167
112
- if (j + vec_elem_num == hidden_size) {
113
- elems_int8.save(output + i * hidden_size + j);
114
- } else {
115
- elems_int8.save(output + i * hidden_size + j, hidden_size - j);
168
+ if constexpr (AZP) {
169
+ elems_fp32 = elems_fp32 + azp_vec;
170
}
171
+ elems_fp32 = elems_fp32.clamp(i8_min_vec, i8_max_vec);
172
+ vec_op::INT8Vec16 elems_int8(elems_fp32);
173
+ elems_int8.save(output + i * hidden_size + j, hidden_size - j);
174
}
175
}
176
}
177
121
-template <bool Bias, typename scalar_t>
122
-void dynamic_output_scale_impl(const float* input, scalar_t* output,
123
- const float* scale, const scalar_t* bias,
124
- const int num_tokens, const int hidden_size) {
178
+template <bool PerChannel, typename scalar_t>
179
+void static_quant_epilogue(const float* input, scalar_t* output,
180
+ const float a_scale, const float* b_scale,
181
+ const int32_t* azp_with_adj, const int num_tokens,
182
+ const int hidden_size) {
183
CPU_KERNEL_GUARD_IN(dynamic_output_scale_impl)
184
using load_vec_t = typename KernelVecType<scalar_t>::load_vec_type;
185
+ using azp_adj_load_vec_t =
186
+ typename KernelVecType<scalar_t>::azp_adj_load_vec_type;
187
+ using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
188
+ constexpr int vec_elem_num = load_vec_t::VEC_ELEM_NUM;
189
+
190
+ #pragma omp parallel for
191
+ for (int i = 0; i < num_tokens; ++i) {
192
+ cvt_vec_t a_scale_vec(a_scale);
193
+ cvt_vec_t b_scale_vec(*b_scale);
194
+ cvt_vec_t scale_vec = a_scale_vec * b_scale_vec;
195
+
196
+ int j = 0;
197
+ for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
198
+ cvt_vec_t elems_fp32(input + i * hidden_size + j);
199
+ azp_adj_load_vec_t azp_adj_vec(azp_with_adj + j);
200
+ cvt_vec_t azp_adj_fp32(azp_adj_vec);
201
+
202
+ if constexpr (PerChannel) {
203
+ b_scale_vec = cvt_vec_t(b_scale + j);
204
+ scale_vec = b_scale_vec * a_scale_vec;
205
+ }
206
+
207
+ elems_fp32 = elems_fp32 - scale_vec * azp_adj_fp32;
208
+
209
+ load_vec_t elems_out(elems_fp32);
210
+ elems_out.save(output + i * hidden_size + j);
211
+ }
212
+
213
+ cvt_vec_t elems_fp32(input + i * hidden_size + j);
214
+ azp_adj_load_vec_t azp_adj_vec(azp_with_adj + j);
215
+ cvt_vec_t azp_adj_fp32(azp_adj_vec);
216
+
217
+ if constexpr (PerChannel) {
218
+ b_scale_vec = cvt_vec_t(b_scale + j);
219
+ scale_vec = b_scale_vec * a_scale_vec;
220
+ }
221
+
222
+ elems_fp32 = elems_fp32 - scale_vec * azp_adj_fp32;
223
+
224
+ load_vec_t elems_out(elems_fp32);
225
+ elems_out.save(output + i * hidden_size + j, hidden_size - j);
226
+ }
227
+}
228
+
229
+template <bool AZP, bool PerChannel, bool Bias, typename scalar_t>
230
+void dynamic_quant_epilogue(const float* input, scalar_t* output,
231
+ const float* a_scale, const float* b_scale,
232
+ const int32_t* azp, const int32_t* azp_adj,
233
+ const scalar_t* bias, const int num_tokens,
234
+ const int hidden_size) {
235
+ CPU_KERNEL_GUARD_IN(dynamic_quant_epilogue)
236
+ using load_vec_t = typename KernelVecType<scalar_t>::load_vec_type;
237
+ using azp_adj_load_vec_t =
238
+ typename KernelVecType<scalar_t>::azp_adj_load_vec_type;
239
using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
240
constexpr int vec_elem_num = load_vec_t::VEC_ELEM_NUM;
241
242
#pragma omp parallel for
243
for (int i = 0; i < num_tokens; ++i) {
244
int j = 0;
133
- cvt_vec_t token_scale_vec(scale[i]);
245
+ cvt_vec_t token_scale_vec(a_scale[i]);
246
+ cvt_vec_t token_zp_scale_vec;
247
+ if constexpr (AZP) {
248
+ float zp_scale_val = a_scale[i] * static_cast<float>(azp[i]);
249
+ if constexpr (!PerChannel) {
250
+ zp_scale_val *= *b_scale;
251
+ }
252
+ token_zp_scale_vec = cvt_vec_t(zp_scale_val);
253
+ }
254
+
255
for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
256
cvt_vec_t elems_fp32(input + i * hidden_size + j);
257
elems_fp32 = elems_fp32 * token_scale_vec;
258
259
+ if constexpr (AZP) {
260
+ azp_adj_load_vec_t azp_adj_vec(azp_adj + j);
261
+ cvt_vec_t azp_adj_fp32(azp_adj_vec);
262
+ azp_adj_fp32 = azp_adj_fp32 * token_zp_scale_vec;
263
+
264
+ if constexpr (PerChannel) {
265
+ cvt_vec_t b_scale_vec(b_scale + j);
266
+ azp_adj_fp32 = azp_adj_fp32 * b_scale_vec;
267
+ }
268
+
269
+ elems_fp32 = elems_fp32 - azp_adj_fp32;
270
+ }
271
+
272
if constexpr (Bias) {
273
load_vec_t bias_vec(bias + j);
274
cvt_vec_t bias_vec_fp32(bias_vec);
@@ -148,6 +282,19 @@ void dynamic_output_scale_impl(const float* input, scalar_t* output,
282
cvt_vec_t elems_fp32(input + i * hidden_size + j);
283
elems_fp32 = elems_fp32 * token_scale_vec;
284
285
+ if constexpr (AZP) {
286
+ azp_adj_load_vec_t azp_adj_vec(azp_adj + j);
287
+ cvt_vec_t azp_adj_fp32(azp_adj_vec);
288
+ azp_adj_fp32 = azp_adj_fp32 * token_zp_scale_vec;
289
+
290
+ if constexpr (PerChannel) {
291
+ cvt_vec_t b_scale_vec(b_scale + j);
292
+ azp_adj_fp32 = azp_adj_fp32 * b_scale_vec;
293
+ }
294
+
295
+ elems_fp32 = elems_fp32 - azp_adj_fp32;
296
+ }
297
+
298
if constexpr (Bias) {
299
load_vec_t bias_vec(bias + j);
300
cvt_vec_t bias_vec_fp32(bias_vec);
@@ -155,32 +302,41 @@ void dynamic_output_scale_impl(const float* input, scalar_t* output,
302
}
303
304
load_vec_t elems_out(elems_fp32);
158
-
159
- if (j + vec_elem_num == hidden_size) {
160
- elems_out.save(output + i * hidden_size + j);
161
- } else {
162
- elems_out.save(output + i * hidden_size + j, hidden_size - j);
163
- }
305
+ elems_out.save(output + i * hidden_size + j, hidden_size - j);
306
}
307
}
308
#else
309
template <typename scalar_t>
310
void static_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
169
- const float* scale, const int num_tokens,
311
+ const float* scale, const int32_t* azp,
312
+ const int num_tokens,
313
const int hidden_size) {
314
TORCH_CHECK(false, "static_scaled_int8_quant_impl requires AVX512 support.")
315
}
316
317
template <typename scalar_t>
318
void dynamic_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
176
- float* scale, const int num_tokens,
319
+ float* scale, int32_t* azp,
320
+ const int num_tokens,
321
const int hidden_size) {
322
TORCH_CHECK(false, "dynamic_scaled_int8_quant_impl requires AVX512 support.")
323
}
324
325
+template <bool PerChannel, typename scalar_t>
326
+void static_quant_epilogue(const float* input, scalar_t* output,
327
+ const float a_scale, const float* b_scale,
328
+ const int32_t* azp_with_adj, const int num_tokens,
329
+ const int hidden_size) {
330
+ TORCH_CHECK(false, "static_quant_epilogue requires AVX512 support.")
331
+}
332
+
333
template <typename scalar_t>
182
-void dynamic_output_scale_impl() {
183
- TORCH_CHECK(false, "dynamic_output_scale_impl requires AVX512 support.")
334
+void dynamic_quant_epilogue(const float* input, scalar_t* output,
335
+ const float* a_scale, const float* b_scale,
336
+ const int32_t* azp, const int32_t* azp_with_adj,
337
+ const scalar_t* bias, const int num_tokens,
338
+ const int hidden_size) {
339
+ TORCH_CHECK(false, "dynamic_quant_epilogue requires AVX512 support.")
340
}
341
#endif
342
} // namespace
@@ -214,39 +370,52 @@ void int8_scaled_mm(torch::Tensor& c, // [M, OC], row-major
370
bias->dim() == 1);
371
}
372
217
- VLLM_DISPATCH_FLOATING_TYPES(c.scalar_type(), "cutlass_scaled_mm", [&] {
373
+ VLLM_DISPATCH_FLOATING_TYPES(c.scalar_type(), "int8_scaled_mm", [&] {
374
if (a_scales.numel() != 1) {
375
// per-token
376
// Note: oneDNN doesn't support per-token activation quantization
377
+ // Ideally we want to fuse the GEMM and the scale procedure with oneDNN
378
+ // JIT, the intermediate data is cached in registers or L1. But for now
379
+ // the oneDNN GEMM code generation only supports two quantization
380
+ // patterns: per-tensor or per-output-channel of weight.
381
+ // So we have to apply the per-token scale with a 'epilogue'. In C=s_a *
382
+ // s_b * (A@B) + bias, the C_inter = s_b * (A@B) is computed by oneDNN
383
+ // GEMM, then the per-token scale (and bias) is applied with the epilogue
384
+ // C=s_a * C_inter + bias.
385
torch::Tensor tmp_fp32_out =
386
torch::empty_like(c, ::at::ScalarType::Float);
223
- DNNLPrimitiveHelper<true>::gemm_s8s8_jit(
387
+ // Compute C_inter=s_b * (A@B)
388
+ DNNLPrimitiveHelper<true>::gemm_s8s8_jit<float, void>(
389
a.data_ptr<int8_t>(), b.data_ptr<int8_t>(),
225
- tmp_fp32_out.data_ptr<float>(), (void*)(0), a.size(0), b.size(1),
226
- a.size(1), (float*)(0), b_scales.data_ptr<float>(), 0,
227
- b_scales.numel());
390
+ tmp_fp32_out.data_ptr<float>(), nullptr, a.size(0), b.size(1),
391
+ a.size(1), nullptr, b_scales.data_ptr<float>(), 0, b_scales.numel());
392
if (bias.has_value()) {
229
- dynamic_output_scale_impl<true>(
393
+ // Compute C=s_a * C_inter + bias
394
+ dynamic_quant_epilogue<false, true, true>(
395
tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
231
- a_scales.data_ptr<float>(), bias->data_ptr<scalar_t>(), c.size(0),
232
- c.size(1));
396
+ a_scales.data_ptr<float>(), nullptr, nullptr, nullptr,
397
+ bias->data_ptr<scalar_t>(), c.size(0), c.size(1));
398
} else {
234
- dynamic_output_scale_impl<false>(
399
+ // Compute C=s_a * C_inter
400
+ dynamic_quant_epilogue<false, true, false, scalar_t>(
401
tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
236
- a_scales.data_ptr<float>(), (scalar_t*)(0), c.size(0), c.size(1));
402
+ a_scales.data_ptr<float>(), nullptr, nullptr, nullptr, nullptr,
403
+ c.size(0), c.size(1));
404
}
405
} else {
406
// per-tensor
407
if (bias.has_value()) {
408
+ // Compute C=s_a * s_b * (A@B) + bias
409
DNNLPrimitiveHelper<false>::gemm_s8s8_jit(
410
a.data_ptr<int8_t>(), b.data_ptr<int8_t>(), c.data_ptr<scalar_t>(),
411
bias->data_ptr<scalar_t>(), a.size(0), b.size(1), a.size(1),
412
a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
413
a_scales.numel(), b_scales.numel());
414
} else {
247
- DNNLPrimitiveHelper<false>::gemm_s8s8_jit(
415
+ // Compute C=s_a * s_b * (A@B)
416
+ DNNLPrimitiveHelper<false>::gemm_s8s8_jit<scalar_t, void>(
417
a.data_ptr<int8_t>(), b.data_ptr<int8_t>(), c.data_ptr<scalar_t>(),
249
- (void*)(0), a.size(0), b.size(1), a.size(1),
418
+ nullptr, a.size(0), b.size(1), a.size(1),
419
a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
420
a_scales.numel(), b_scales.numel());
421
}
@@ -254,6 +423,127 @@ void int8_scaled_mm(torch::Tensor& c, // [M, OC], row-major
423
});
424
}
425
426
+void int8_scaled_mm_azp(torch::Tensor& c, // [M, OC], row-major
427
+ const torch::Tensor& a, // [M, IC], row-major
428
+ const torch::Tensor& b, // [IC, OC], column-major
429
+ const torch::Tensor& a_scales, // [1] or [M]
430
+ const torch::Tensor& b_scales, // [1] or [OC]
431
+ const torch::Tensor& azp_adj, // [OC]
432
+ const c10::optional<torch::Tensor>& azp, // [1] or [M]
433
+ const c10::optional<torch::Tensor>& bias // [OC]
434
+) {
435
+ CPU_KERNEL_GUARD_IN(cutlass_scaled_mm_azp)
436
+ // Checks for conformality
437
+ TORCH_CHECK(a.dtype() == torch::kInt8 && b.dtype() == torch::kInt8,
438
+ "int8_scaled_mm_azp only supports INT8 inputs.")
439
+ TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
440
+ TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
441
+ b.size(1) == c.size(1));
442
+ TORCH_CHECK(a_scales.numel() == 1 || a_scales.numel() == a.size(0));
443
+ TORCH_CHECK(b_scales.numel() == 1 || b_scales.numel() == b.size(1));
444
+
445
+ // Check for strides and alignment
446
+ TORCH_CHECK(a.stride(1) == 1 && c.stride(1) == 1); // Row-major
447
+ TORCH_CHECK(b.stride(0) == 1); // Column-major
448
+ TORCH_CHECK(c.stride(0) % 16 == 0 &&
449
+ b.stride(1) % 16 == 0); // 16 Byte Alignment
450
+ TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
451
+
452
+ if (bias) {
453
+ TORCH_CHECK(bias->numel() == b.size(1) && bias->is_contiguous());
454
+ }
455
+ if (azp) {
456
+ TORCH_CHECK(azp->numel() == a.size(0) && azp->is_contiguous());
457
+ }
458
+ TORCH_CHECK(azp_adj.numel() == b.size(1) && azp_adj.is_contiguous());
459
+
460
+ // azp & bias types
461
+ TORCH_CHECK(azp_adj.dtype() == torch::kInt32);
462
+ TORCH_CHECK(!azp || azp->dtype() == torch::kInt32);
463
+ TORCH_CHECK(!bias || bias->dtype() == c.dtype(),
464
+ "currently bias dtype must match output dtype ", c.dtype());
465
+
466
+ VLLM_DISPATCH_FLOATING_TYPES(c.scalar_type(), "int8_scaled_mm_azp", [&] {
467
+ torch::Tensor tmp_fp32_out = torch::empty_like(c, ::at::ScalarType::Float);
468
+ if (a_scales.numel() != 1) {
469
+ // per-token
470
+ // Note: oneDNN doesn't support per-token activation quantization
471
+ // Compute C_inter=s_b * (A@B)
472
+ DNNLPrimitiveHelper<true>::gemm_s8s8_jit<float, void>(
473
+ a.data_ptr<int8_t>(), b.data_ptr<int8_t>(),
474
+ tmp_fp32_out.data_ptr<float>(), nullptr, a.size(0), b.size(1),
475
+ a.size(1), nullptr, b_scales.data_ptr<float>(), 0, b_scales.numel());
476
+ if (bias.has_value()) {
477
+ // Compute C=s_a * C_inter - s_a * s_b * azp * azp_adj + bias
478
+ if (b_scales.numel() != 1) {
479
+ // Per-Channel
480
+ dynamic_quant_epilogue<true, true, true>(
481
+ tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
482
+ a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
483
+ azp->data_ptr<int32_t>(), azp_adj.data_ptr<int32_t>(),
484
+ bias->data_ptr<scalar_t>(), c.size(0), c.size(1));
485
+ } else {
486
+ // Per-Tensor
487
+ dynamic_quant_epilogue<true, false, true>(
488
+ tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
489
+ a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
490
+ azp->data_ptr<int32_t>(), azp_adj.data_ptr<int32_t>(),
491
+ bias->data_ptr<scalar_t>(), c.size(0), c.size(1));
492
+ }
493
+ } else {
494
+ // Compute C=s_a * C_inter - s_a * s_b * azp * azp_adj
495
+ if (b_scales.numel() != 1) {
496
+ // Per-Channel
497
+ dynamic_quant_epilogue<true, true, false, scalar_t>(
498
+ tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
499
+ a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
500
+ azp->data_ptr<int32_t>(), azp_adj.data_ptr<int32_t>(), nullptr,
501
+ c.size(0), c.size(1));
502
+ } else {
503
+ // Per-Tensor
504
+ dynamic_quant_epilogue<true, false, false, scalar_t>(
505
+ tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
506
+ a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
507
+ azp->data_ptr<int32_t>(), azp_adj.data_ptr<int32_t>(), nullptr,
508
+ c.size(0), c.size(1));
509
+ }
510
+ }
511
+ } else {
512
+ // per-tensor
513
+ if (bias.has_value()) {
514
+ // Compute C_inter=s_a * s_b * (A@B) + bias
515
+ DNNLPrimitiveHelper<false>::gemm_s8s8_jit(
516
+ a.data_ptr<int8_t>(), b.data_ptr<int8_t>(),
517
+ tmp_fp32_out.data_ptr<float>(), bias->data_ptr<scalar_t>(),
518
+ a.size(0), b.size(1), a.size(1), a_scales.data_ptr<float>(),
519
+ b_scales.data_ptr<float>(), a_scales.numel(), b_scales.numel());
520
+ } else {
521
+ // Compute C_inter=s_a * s_b * (A@B)
522
+ DNNLPrimitiveHelper<false>::gemm_s8s8_jit<float, void>(
523
+ a.data_ptr<int8_t>(), b.data_ptr<int8_t>(),
524
+ tmp_fp32_out.data_ptr<float>(), nullptr, a.size(0), b.size(1),
525
+ a.size(1), a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
526
+ a_scales.numel(), b_scales.numel());
527
+ }
528
+
529
+ // Compute C=C_inter - s_a * s_b * azp_adj
530
+ if (b_scales.numel() != 1) {
531
+ // Per-Channel
532
+ static_quant_epilogue<true>(
533
+ tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
534
+ *a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
535
+ azp_adj.data_ptr<int32_t>(), a.size(0), b.size(1));
536
+ } else {
537
+ // Per-Tensor
538
+ static_quant_epilogue<false>(
539
+ tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
540
+ *a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
541
+ azp_adj.data_ptr<int32_t>(), a.size(0), b.size(1));
542
+ }
543
+ }
544
+ });
545
+}
546
+
547
// static-per-tensor quantization.
548
void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
549
const torch::Tensor& input, // [..., hidden_size]
@@ -263,15 +553,22 @@ void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
553
TORCH_CHECK(input.is_contiguous());
554
TORCH_CHECK(out.is_contiguous());
555
TORCH_CHECK(scale.numel() == 1);
266
- TORCH_CHECK(!azp.has_value(), "Zero point is not supported on CPU.");
556
+ TORCH_CHECK(!azp.has_value() || azp->numel() == 1);
557
558
const int hidden_size = input.size(-1);
559
const int num_tokens = input.numel() / hidden_size;
560
VLLM_DISPATCH_FLOATING_TYPES(
561
input.scalar_type(), "static_scaled_int8_quant_impl", [&] {
272
- static_scaled_int8_quant_impl(
273
- input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
274
- scale.data_ptr<float>(), num_tokens, hidden_size);
562
+ if (azp.has_value()) {
563
+ static_scaled_int8_quant_impl<true>(
564
+ input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
565
+ scale.data_ptr<float>(), azp->data_ptr<int32_t>(), num_tokens,
566
+ hidden_size);
567
+ } else {
568
+ static_scaled_int8_quant_impl<false>(
569
+ input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
570
+ scale.data_ptr<float>(), nullptr, num_tokens, hidden_size);
571
+ }
572
});
573
}
574
@@ -284,14 +581,20 @@ void dynamic_scaled_int8_quant(
581
CPU_KERNEL_GUARD_IN(dynamic_scaled_int8_quant)
582
TORCH_CHECK(input.is_contiguous());
583
TORCH_CHECK(out.is_contiguous());
287
- TORCH_CHECK(!azp.has_value(), "Zero point is not supported on CPU.");
584
585
int const hidden_size = input.size(-1);
586
int const num_tokens = input.numel() / hidden_size;
587
VLLM_DISPATCH_FLOATING_TYPES(
588
input.scalar_type(), "dynamic_scaled_int8_quant_impl", [&] {
293
- dynamic_scaled_int8_quant_impl(
294
- input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
295
- scale.data_ptr<float>(), num_tokens, hidden_size);
589
+ if (azp.has_value()) {
590
+ dynamic_scaled_int8_quant_impl<true>(
591
+ input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
592
+ scale.data_ptr<float>(), azp->data_ptr<int32_t>(), num_tokens,
593
+ hidden_size);
594
+ } else {
595
+ dynamic_scaled_int8_quant_impl<false>(
596
+ input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
597
+ scale.data_ptr<float>(), nullptr, num_tokens, hidden_size);
598
+ }
599
});
600
}
csrc/cpu/torch_bindings.cpp
+15
@@ -11,6 +11,13 @@ void int8_scaled_mm(torch::Tensor& c, const torch::Tensor& a,
11
const torch::Tensor& b_scales,
12
const c10::optional<torch::Tensor>& bias);
13
14
+void int8_scaled_mm_azp(torch::Tensor& c, const torch::Tensor& a,
15
+ const torch::Tensor& b, const torch::Tensor& a_scales,
16
+ const torch::Tensor& b_scales,
17
+ const torch::Tensor& azp_adj,
18
+ const c10::optional<torch::Tensor>& azp,
19
+ const c10::optional<torch::Tensor>& bias);
20
+
21
TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
22
// vLLM custom ops
23
@@ -111,6 +118,14 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
118
" Tensor b, Tensor a_scales,"
119
" Tensor b_scales, Tensor? bias) -> ()");
120
ops.impl("cutlass_scaled_mm", torch::kCPU, &int8_scaled_mm);
121
+ // w8a8 GEMM, supporting asymmetric per-tensor or per-row/column
122
+ // quantization.
123
+ ops.def(
124
+ "cutlass_scaled_mm_azp(Tensor! out, Tensor a,"
125
+ " Tensor b, Tensor a_scales,"
126
+ " Tensor b_scales, Tensor azp_adj,"
127
+ " Tensor? azp, Tensor? bias) -> ()");
128
+ ops.impl("cutlass_scaled_mm_azp", torch::kCPU, &int8_scaled_mm_azp);
129
#endif
130
}
131
docs/source/getting_started/cpu-installation.rst
-14
@@ -59,20 +59,6 @@ Build from source
59
$ pip install cmake>=3.26 wheel packaging ninja "setuptools-scm>=8" numpy
60
$ pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu
61
62
-- Third, build and install oneDNN library from source:
63
-
64
-.. code-block:: console
65
-
66
- $ git clone -b rls-v3.5 https://github.com/oneapi-src/oneDNN.git
67
- $ cmake -B ./oneDNN/build -S ./oneDNN -G Ninja -DONEDNN_LIBRARY_TYPE=STATIC \
68
- -DONEDNN_BUILD_DOC=OFF \
69
- -DONEDNN_BUILD_EXAMPLES=OFF \
70
- -DONEDNN_BUILD_TESTS=OFF \
71
- -DONEDNN_BUILD_GRAPH=OFF \
72
- -DONEDNN_ENABLE_WORKLOAD=INFERENCE \
73
- -DONEDNN_ENABLE_PRIMITIVE=MATMUL
74
- $ cmake --build ./oneDNN/build --target install --config Release
75
-
62
- Finally, build and install vLLM CPU backend:
63
64
.. code-block:: console