@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5eda21e773
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Hardware][CPU] compressed-tensor INT8 W8A8 AZP support (#9344)

Li, Jiang committed Oct 18, 2024 at 00:21 UTC 5eda21e773447d81ffc661ac094716420dc7b7cb
7 files changed +452 -96
.buildkite/run-cpu-test.sh
+4 -4
@@ -32,10 +32,10 @@ docker exec cpu-test bash -c "
32 --ignore=tests/models/decoder_only/language/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
33
34 # Run compressed-tensor test
35 -# docker exec cpu-test bash -c "
36 -# pytest -s -v \
37 -# tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_static_setup \
38 -# tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_dynanmic_per_token"
35 +docker exec cpu-test bash -c "
36 + pytest -s -v \
37 + tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_static_setup \
38 + tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_dynamic_per_token"
39
40 # Run AWQ test
41 docker exec cpu-test bash -c "
Dockerfile.cpu
-13
@@ -33,19 +33,6 @@ RUN --mount=type=cache,target=/root/.cache/pip \
33 pip install --upgrade pip && \
34 pip install -r requirements-build.txt
35
36 -# install oneDNN
37 -RUN git clone -b rls-v3.5 https://github.com/oneapi-src/oneDNN.git
38 -
39 -RUN --mount=type=cache,target=/root/.cache/ccache \
40 - cmake -B ./oneDNN/build -S ./oneDNN -G Ninja -DONEDNN_LIBRARY_TYPE=STATIC \
41 - -DONEDNN_BUILD_DOC=OFF \
42 - -DONEDNN_BUILD_EXAMPLES=OFF \
43 - -DONEDNN_BUILD_TESTS=OFF \
44 - -DONEDNN_BUILD_GRAPH=OFF \
45 - -DONEDNN_ENABLE_WORKLOAD=INFERENCE \
46 - -DONEDNN_ENABLE_PRIMITIVE=MATMUL && \
47 - cmake --build ./oneDNN/build --target install --config Release
48 -
36 FROM cpu-test-1 AS build
37
38 WORKDIR /workspace/vllm
cmake/cpu_extension.cmake
+34 -6
@@ -1,5 +1,8 @@
1 +include(FetchContent)
2 +
3 +set(CMAKE_CXX_STANDARD_REQUIRED ON)
4 +set(CMAKE_CXX_EXTENSIONS ON)
5 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
2 -set(CMAKE_CXX_STANDARD 17)
6
7 #
8 # Define environment variables for special configurations
@@ -82,15 +85,40 @@ else()
85 message(FATAL_ERROR "vLLM CPU backend requires AVX512 or AVX2 or Power9+ ISA support.")
86 endif()
87
88 +#
89 +# Build oneDNN for W8A8 GEMM kernels (only for x86-AVX512 platforms)
90 +#
91 +if (AVX512_FOUND AND NOT AVX512_DISABLED)
92 + FetchContent_Declare(
93 + oneDNN
94 + GIT_REPOSITORY https://github.com/oneapi-src/oneDNN.git
95 + GIT_TAG v3.5.3
96 + GIT_PROGRESS TRUE
97 + GIT_SHALLOW TRUE
98 + )
99 +
100 + set(ONEDNN_LIBRARY_TYPE "STATIC")
101 + set(ONEDNN_BUILD_DOC "OFF")
102 + set(ONEDNN_BUILD_EXAMPLES "OFF")
103 + set(ONEDNN_BUILD_TESTS "OFF")
104 + set(ONEDNN_ENABLE_WORKLOAD "INFERENCE")
105 + set(ONEDNN_ENABLE_PRIMITIVE "MATMUL;REORDER")
106 + set(ONEDNN_BUILD_GRAPH "OFF")
107 + set(ONEDNN_ENABLE_JIT_PROFILING "OFF")
108 + set(ONEDNN_ENABLE_ITT_TASKS "OFF")
109 + set(ONEDNN_ENABLE_MAX_CPU_ISA "OFF")
110 + set(ONEDNN_ENABLE_CPU_ISA_HINTS "OFF")
111 + set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
112 +
113 + FetchContent_MakeAvailable(oneDNN)
114 +
115 + list(APPEND LIBS dnnl)
116 +endif()
117 +
118 message(STATUS "CPU extension compile flags: ${CXX_COMPILE_FLAGS}")
119
120 list(APPEND LIBS numa)
121
89 -# Appending the dnnl library for the AVX2 and AVX512, as it is not utilized by Power architecture.
90 -if (AVX2_FOUND OR AVX512_FOUND)
91 - list(APPEND LIBS dnnl)
92 -endif()
93 -
122 #
123 # _C extension
124 #
csrc/cpu/cpu_types_x86.hpp
+39 -2
@@ -265,6 +265,30 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
265 void save(float *ptr) const { _mm256_storeu_ps(ptr, reg); }
266 };
267
268 +#ifdef __AVX512F__
269 +struct INT32Vec16: public Vec<INT32Vec16> {
270 + constexpr static int VEC_ELEM_NUM = 16;
271 + union AliasReg {
272 + __m512i reg;
273 + int32_t values[VEC_ELEM_NUM];
274 + };
275 +
276 + __m512i reg;
277 +
278 + explicit INT32Vec16(const void* data_ptr) : reg(_mm512_loadu_epi32(data_ptr)) {}
279 +
280 + void save(int32_t* ptr) const {
281 + _mm512_storeu_epi32(ptr, reg);
282 + }
283 +
284 + void save(int32_t* ptr, const int elem_num) const {
285 + constexpr uint32_t M = 0xFFFFFFFF;
286 + __mmask16 mask = _cvtu32_mask16(M >> (32 - elem_num));
287 + _mm512_mask_storeu_epi32(ptr, mask, reg);
288 + }
289 +};
290 +#endif
291 +
292 #ifdef __AVX512F__
293 struct FP32Vec16 : public Vec<FP32Vec16> {
294 constexpr static int VEC_ELEM_NUM = 16;
@@ -283,8 +307,6 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
307
308 explicit FP32Vec16(__m512 data) : reg(data) {}
309
286 - explicit FP32Vec16(const FP32Vec16 &data) : reg(data.reg) {}
287 -
310 explicit FP32Vec16(const FP32Vec4 &data)
311 : reg((__m512)_mm512_inserti32x4(
312 _mm512_inserti32x4(
@@ -303,6 +325,9 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
325
326 explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
327
328 + explicit FP32Vec16(const INT32Vec16 &v)
329 + : reg(_mm512_cvt_roundepi32_ps(v.reg, _MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC)) {}
330 +
331 FP32Vec16 operator*(const FP32Vec16 &b) const {
332 return FP32Vec16(_mm512_mul_ps(reg, b.reg));
333 }
@@ -333,6 +358,16 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
358 return FP32Vec16(_mm512_mask_max_ps(reg, mask, reg, b.reg));
359 }
360
361 + FP32Vec16 min(const FP32Vec16& b) const {
362 + return FP32Vec16(_mm512_min_ps(reg, b.reg));
363 + }
364 +
365 + FP32Vec16 min(const FP32Vec16& b, const int elem_num) const {
366 + constexpr uint32_t M = 0xFFFFFFFF;
367 + __mmask16 mask = _cvtu32_mask16(M >> (32 - elem_num));
368 + return FP32Vec16(_mm512_mask_min_ps(reg, mask, reg, b.reg));
369 + }
370 +
371 FP32Vec16 abs() const {
372 return FP32Vec16(_mm512_abs_ps(reg));
373 }
@@ -341,6 +376,8 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
376
377 float reduce_max() const { return _mm512_reduce_max_ps(reg); }
378
379 + float reduce_min() const { return _mm512_reduce_min_ps(reg); }
380 +
381 template <int group_size> float reduce_sub_sum(int idx) {
382 static_assert(VEC_ELEM_NUM % group_size == 0);
383 constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
csrc/cpu/quant.cpp
+360 -57
@@ -5,25 +5,29 @@ namespace {
5 template <typename scalar_t>
6 struct KernelVecType {
7 using load_vec_type = void;
8 + using azp_adj_load_vec_type = void;
9 using cvt_vec_type = void;
10 };
11
12 template <>
13 struct KernelVecType<float> {
14 using load_vec_type = vec_op::FP32Vec16;
15 + using azp_adj_load_vec_type = vec_op::INT32Vec16;
16 using cvt_vec_type = vec_op::FP32Vec16;
17 };
18
19 template <>
20 struct KernelVecType<c10::BFloat16> {
21 using load_vec_type = vec_op::BF16Vec16;
22 + using azp_adj_load_vec_type = vec_op::INT32Vec16;
23 using cvt_vec_type = vec_op::FP32Vec16;
24 };
25
26 #ifdef __AVX512F__
24 -template <typename scalar_t>
27 +template <bool AZP, typename scalar_t>
28 void static_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
26 - const float* scale, const int num_tokens,
29 + const float* scale, const int32_t* azp,
30 + const int num_tokens,
31 const int hidden_size) {
32 using load_vec_t = typename KernelVecType<scalar_t>::load_vec_type;
33 using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
@@ -37,62 +41,110 @@ void static_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
41 const cvt_vec_t i8_min_vec(i8_min);
42 const cvt_vec_t i8_max_vec(i8_max);
43
44 + cvt_vec_t zp_vec;
45 + if constexpr (AZP) {
46 + zp_vec = cvt_vec_t(static_cast<float>(*azp));
47 + }
48 +
49 #pragma omp parallel for
50 for (int i = 0; i < num_tokens; ++i) {
51 int j = 0;
52 for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
53 load_vec_t elems(input + i * hidden_size + j);
54 cvt_vec_t elems_fp32(elems);
46 - elems_fp32 = (elems_fp32 * inv_scale).clamp(i8_min_vec, i8_max_vec);
55 + elems_fp32 = elems_fp32 * inv_scale;
56 +
57 + if constexpr (AZP) {
58 + elems_fp32 = elems_fp32 + zp_vec;
59 + }
60 +
61 + elems_fp32 = elems_fp32.clamp(i8_min_vec, i8_max_vec);
62 vec_op::INT8Vec16 elems_int8(elems_fp32);
63 elems_int8.save(output + i * hidden_size + j);
64 }
65
66 load_vec_t elems(input + i * hidden_size + j);
67 cvt_vec_t elems_fp32(elems);
53 - elems_fp32 = (elems_fp32 * inv_scale).clamp(i8_min_vec, i8_max_vec);
54 - vec_op::INT8Vec16 elems_int8(elems_fp32);
68 + elems_fp32 = elems_fp32 * inv_scale;
69
56 - if (j + vec_elem_num == hidden_size) {
57 - elems_int8.save(output + i * hidden_size + j);
58 - } else {
59 - elems_int8.save(output + i * hidden_size + j, hidden_size - j);
70 + if constexpr (AZP) {
71 + elems_fp32 = elems_fp32 + zp_vec;
72 }
73 +
74 + elems_fp32 = elems_fp32.clamp(i8_min_vec, i8_max_vec);
75 + vec_op::INT8Vec16 elems_int8(elems_fp32);
76 + elems_int8.save(output + i * hidden_size + j, hidden_size - j);
77 }
78 }
79
64 -template <typename scalar_t>
80 +template <bool AZP, typename scalar_t>
81 void dynamic_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
66 - float* scale, const int num_tokens,
82 + float* scale, int32_t* azp,
83 + const int num_tokens,
84 const int hidden_size) {
85 using load_vec_t = typename KernelVecType<scalar_t>::load_vec_type;
86 using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
87 constexpr int vec_elem_num = load_vec_t::VEC_ELEM_NUM;
88
89 + constexpr float i8_min =
90 + static_cast<float>(std::numeric_limits<int8_t>::min());
91 + constexpr float i8_max =
92 + static_cast<float>(std::numeric_limits<int8_t>::max());
93 + const cvt_vec_t i8_min_vec(i8_min);
94 + const cvt_vec_t i8_max_vec(i8_max);
95 +
96 #pragma omp parallel for
97 for (int i = 0; i < num_tokens; ++i) {
74 - cvt_vec_t max_abs(0.0);
98 + cvt_vec_t max_value(std::numeric_limits<float>::lowest());
99 + cvt_vec_t min_value(std::numeric_limits<float>::max());
100 {
101 int j = 0;
102 for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
103 load_vec_t elems(input + i * hidden_size + j);
104 cvt_vec_t elems_fp32(elems);
80 - max_abs = max_abs.max(elems_fp32.abs());
105 + if constexpr (AZP) {
106 + max_value = max_value.max(elems_fp32);
107 + min_value = min_value.min(elems_fp32);
108 + } else {
109 + max_value = max_value.max(elems_fp32.abs());
110 + }
111 }
112
113 load_vec_t elems(input + i * hidden_size + j);
114 cvt_vec_t elems_fp32(elems);
115
116 if (j + vec_elem_num == hidden_size) {
87 - max_abs = max_abs.max(elems_fp32.abs());
117 + if constexpr (AZP) {
118 + max_value = max_value.max(elems_fp32);
119 + min_value = min_value.min(elems_fp32);
120 + } else {
121 + max_value = max_value.max(elems_fp32.abs());
122 + }
123 } else {
89 - max_abs = max_abs.max(elems_fp32.abs(), hidden_size - j);
124 + if constexpr (AZP) {
125 + max_value = max_value.max(elems_fp32, hidden_size - j);
126 + min_value = min_value.min(elems_fp32, hidden_size - j);
127 + } else {
128 + max_value = max_value.max(elems_fp32.abs(), hidden_size - j);
129 + }
130 }
131 }
132
93 - float scale_val = max_abs.reduce_max() / 127.0f;
94 - scale[i] = scale_val;
133 + float scale_val, azp_val;
134 + if constexpr (AZP) {
135 + float max_scalar = max_value.reduce_max();
136 + float min_scalar = min_value.reduce_min();
137 + scale_val = (max_scalar - min_scalar) / 255.0f;
138 + azp_val = std::nearbyint(-128.0f - min_scalar / scale_val);
139 + azp[i] = static_cast<int32_t>(azp_val);
140 + scale[i] = scale_val;
141 + } else {
142 + scale_val = max_value.reduce_max() / 127.0f;
143 + scale[i] = scale_val;
144 + }
145 +
146 const cvt_vec_t inv_scale(1.0 / scale_val);
147 + const cvt_vec_t azp_vec(azp_val);
148
149 {
150 int j = 0;
@@ -100,6 +152,11 @@ void dynamic_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
152 load_vec_t elems(input + i * hidden_size + j);
153 cvt_vec_t elems_fp32(elems);
154 elems_fp32 = (elems_fp32 * inv_scale);
155 +
156 + if constexpr (AZP) {
157 + elems_fp32 = elems_fp32 + azp_vec;
158 + }
159 + elems_fp32 = elems_fp32.clamp(i8_min_vec, i8_max_vec);
160 vec_op::INT8Vec16 elems_int8(elems_fp32);
161 elems_int8.save(output + i * hidden_size + j);
162 }
@@ -107,34 +164,111 @@ void dynamic_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
164 load_vec_t elems(input + i * hidden_size + j);
165 cvt_vec_t elems_fp32(elems);
166 elems_fp32 = (elems_fp32 * inv_scale);
110 - vec_op::INT8Vec16 elems_int8(elems_fp32);
167
112 - if (j + vec_elem_num == hidden_size) {
113 - elems_int8.save(output + i * hidden_size + j);
114 - } else {
115 - elems_int8.save(output + i * hidden_size + j, hidden_size - j);
168 + if constexpr (AZP) {
169 + elems_fp32 = elems_fp32 + azp_vec;
170 }
171 + elems_fp32 = elems_fp32.clamp(i8_min_vec, i8_max_vec);
172 + vec_op::INT8Vec16 elems_int8(elems_fp32);
173 + elems_int8.save(output + i * hidden_size + j, hidden_size - j);
174 }
175 }
176 }
177
121 -template <bool Bias, typename scalar_t>
122 -void dynamic_output_scale_impl(const float* input, scalar_t* output,
123 - const float* scale, const scalar_t* bias,
124 - const int num_tokens, const int hidden_size) {
178 +template <bool PerChannel, typename scalar_t>
179 +void static_quant_epilogue(const float* input, scalar_t* output,
180 + const float a_scale, const float* b_scale,
181 + const int32_t* azp_with_adj, const int num_tokens,
182 + const int hidden_size) {
183 CPU_KERNEL_GUARD_IN(dynamic_output_scale_impl)
184 using load_vec_t = typename KernelVecType<scalar_t>::load_vec_type;
185 + using azp_adj_load_vec_t =
186 + typename KernelVecType<scalar_t>::azp_adj_load_vec_type;
187 + using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
188 + constexpr int vec_elem_num = load_vec_t::VEC_ELEM_NUM;
189 +
190 + #pragma omp parallel for
191 + for (int i = 0; i < num_tokens; ++i) {
192 + cvt_vec_t a_scale_vec(a_scale);
193 + cvt_vec_t b_scale_vec(*b_scale);
194 + cvt_vec_t scale_vec = a_scale_vec * b_scale_vec;
195 +
196 + int j = 0;
197 + for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
198 + cvt_vec_t elems_fp32(input + i * hidden_size + j);
199 + azp_adj_load_vec_t azp_adj_vec(azp_with_adj + j);
200 + cvt_vec_t azp_adj_fp32(azp_adj_vec);
201 +
202 + if constexpr (PerChannel) {
203 + b_scale_vec = cvt_vec_t(b_scale + j);
204 + scale_vec = b_scale_vec * a_scale_vec;
205 + }
206 +
207 + elems_fp32 = elems_fp32 - scale_vec * azp_adj_fp32;
208 +
209 + load_vec_t elems_out(elems_fp32);
210 + elems_out.save(output + i * hidden_size + j);
211 + }
212 +
213 + cvt_vec_t elems_fp32(input + i * hidden_size + j);
214 + azp_adj_load_vec_t azp_adj_vec(azp_with_adj + j);
215 + cvt_vec_t azp_adj_fp32(azp_adj_vec);
216 +
217 + if constexpr (PerChannel) {
218 + b_scale_vec = cvt_vec_t(b_scale + j);
219 + scale_vec = b_scale_vec * a_scale_vec;
220 + }
221 +
222 + elems_fp32 = elems_fp32 - scale_vec * azp_adj_fp32;
223 +
224 + load_vec_t elems_out(elems_fp32);
225 + elems_out.save(output + i * hidden_size + j, hidden_size - j);
226 + }
227 +}
228 +
229 +template <bool AZP, bool PerChannel, bool Bias, typename scalar_t>
230 +void dynamic_quant_epilogue(const float* input, scalar_t* output,
231 + const float* a_scale, const float* b_scale,
232 + const int32_t* azp, const int32_t* azp_adj,
233 + const scalar_t* bias, const int num_tokens,
234 + const int hidden_size) {
235 + CPU_KERNEL_GUARD_IN(dynamic_quant_epilogue)
236 + using load_vec_t = typename KernelVecType<scalar_t>::load_vec_type;
237 + using azp_adj_load_vec_t =
238 + typename KernelVecType<scalar_t>::azp_adj_load_vec_type;
239 using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
240 constexpr int vec_elem_num = load_vec_t::VEC_ELEM_NUM;
241
242 #pragma omp parallel for
243 for (int i = 0; i < num_tokens; ++i) {
244 int j = 0;
133 - cvt_vec_t token_scale_vec(scale[i]);
245 + cvt_vec_t token_scale_vec(a_scale[i]);
246 + cvt_vec_t token_zp_scale_vec;
247 + if constexpr (AZP) {
248 + float zp_scale_val = a_scale[i] * static_cast<float>(azp[i]);
249 + if constexpr (!PerChannel) {
250 + zp_scale_val *= *b_scale;
251 + }
252 + token_zp_scale_vec = cvt_vec_t(zp_scale_val);
253 + }
254 +
255 for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
256 cvt_vec_t elems_fp32(input + i * hidden_size + j);
257 elems_fp32 = elems_fp32 * token_scale_vec;
258
259 + if constexpr (AZP) {
260 + azp_adj_load_vec_t azp_adj_vec(azp_adj + j);
261 + cvt_vec_t azp_adj_fp32(azp_adj_vec);
262 + azp_adj_fp32 = azp_adj_fp32 * token_zp_scale_vec;
263 +
264 + if constexpr (PerChannel) {
265 + cvt_vec_t b_scale_vec(b_scale + j);
266 + azp_adj_fp32 = azp_adj_fp32 * b_scale_vec;
267 + }
268 +
269 + elems_fp32 = elems_fp32 - azp_adj_fp32;
270 + }
271 +
272 if constexpr (Bias) {
273 load_vec_t bias_vec(bias + j);
274 cvt_vec_t bias_vec_fp32(bias_vec);
@@ -148,6 +282,19 @@ void dynamic_output_scale_impl(const float* input, scalar_t* output,
282 cvt_vec_t elems_fp32(input + i * hidden_size + j);
283 elems_fp32 = elems_fp32 * token_scale_vec;
284
285 + if constexpr (AZP) {
286 + azp_adj_load_vec_t azp_adj_vec(azp_adj + j);
287 + cvt_vec_t azp_adj_fp32(azp_adj_vec);
288 + azp_adj_fp32 = azp_adj_fp32 * token_zp_scale_vec;
289 +
290 + if constexpr (PerChannel) {
291 + cvt_vec_t b_scale_vec(b_scale + j);
292 + azp_adj_fp32 = azp_adj_fp32 * b_scale_vec;
293 + }
294 +
295 + elems_fp32 = elems_fp32 - azp_adj_fp32;
296 + }
297 +
298 if constexpr (Bias) {
299 load_vec_t bias_vec(bias + j);
300 cvt_vec_t bias_vec_fp32(bias_vec);
@@ -155,32 +302,41 @@ void dynamic_output_scale_impl(const float* input, scalar_t* output,
302 }
303
304 load_vec_t elems_out(elems_fp32);
158 -
159 - if (j + vec_elem_num == hidden_size) {
160 - elems_out.save(output + i * hidden_size + j);
161 - } else {
162 - elems_out.save(output + i * hidden_size + j, hidden_size - j);
163 - }
305 + elems_out.save(output + i * hidden_size + j, hidden_size - j);
306 }
307 }
308 #else
309 template <typename scalar_t>
310 void static_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
169 - const float* scale, const int num_tokens,
311 + const float* scale, const int32_t* azp,
312 + const int num_tokens,
313 const int hidden_size) {
314 TORCH_CHECK(false, "static_scaled_int8_quant_impl requires AVX512 support.")
315 }
316
317 template <typename scalar_t>
318 void dynamic_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
176 - float* scale, const int num_tokens,
319 + float* scale, int32_t* azp,
320 + const int num_tokens,
321 const int hidden_size) {
322 TORCH_CHECK(false, "dynamic_scaled_int8_quant_impl requires AVX512 support.")
323 }
324
325 +template <bool PerChannel, typename scalar_t>
326 +void static_quant_epilogue(const float* input, scalar_t* output,
327 + const float a_scale, const float* b_scale,
328 + const int32_t* azp_with_adj, const int num_tokens,
329 + const int hidden_size) {
330 + TORCH_CHECK(false, "static_quant_epilogue requires AVX512 support.")
331 +}
332 +
333 template <typename scalar_t>
182 -void dynamic_output_scale_impl() {
183 - TORCH_CHECK(false, "dynamic_output_scale_impl requires AVX512 support.")
334 +void dynamic_quant_epilogue(const float* input, scalar_t* output,
335 + const float* a_scale, const float* b_scale,
336 + const int32_t* azp, const int32_t* azp_with_adj,
337 + const scalar_t* bias, const int num_tokens,
338 + const int hidden_size) {
339 + TORCH_CHECK(false, "dynamic_quant_epilogue requires AVX512 support.")
340 }
341 #endif
342 } // namespace
@@ -214,39 +370,52 @@ void int8_scaled_mm(torch::Tensor& c, // [M, OC], row-major
370 bias->dim() == 1);
371 }
372
217 - VLLM_DISPATCH_FLOATING_TYPES(c.scalar_type(), "cutlass_scaled_mm", [&] {
373 + VLLM_DISPATCH_FLOATING_TYPES(c.scalar_type(), "int8_scaled_mm", [&] {
374 if (a_scales.numel() != 1) {
375 // per-token
376 // Note: oneDNN doesn't support per-token activation quantization
377 + // Ideally we want to fuse the GEMM and the scale procedure with oneDNN
378 + // JIT, the intermediate data is cached in registers or L1. But for now
379 + // the oneDNN GEMM code generation only supports two quantization
380 + // patterns: per-tensor or per-output-channel of weight.
381 + // So we have to apply the per-token scale with a 'epilogue'. In C=s_a *
382 + // s_b * (A@B) + bias, the C_inter = s_b * (A@B) is computed by oneDNN
383 + // GEMM, then the per-token scale (and bias) is applied with the epilogue
384 + // C=s_a * C_inter + bias.
385 torch::Tensor tmp_fp32_out =
386 torch::empty_like(c, ::at::ScalarType::Float);
223 - DNNLPrimitiveHelper<true>::gemm_s8s8_jit(
387 + // Compute C_inter=s_b * (A@B)
388 + DNNLPrimitiveHelper<true>::gemm_s8s8_jit<float, void>(
389 a.data_ptr<int8_t>(), b.data_ptr<int8_t>(),
225 - tmp_fp32_out.data_ptr<float>(), (void*)(0), a.size(0), b.size(1),
226 - a.size(1), (float*)(0), b_scales.data_ptr<float>(), 0,
227 - b_scales.numel());
390 + tmp_fp32_out.data_ptr<float>(), nullptr, a.size(0), b.size(1),
391 + a.size(1), nullptr, b_scales.data_ptr<float>(), 0, b_scales.numel());
392 if (bias.has_value()) {
229 - dynamic_output_scale_impl<true>(
393 + // Compute C=s_a * C_inter + bias
394 + dynamic_quant_epilogue<false, true, true>(
395 tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
231 - a_scales.data_ptr<float>(), bias->data_ptr<scalar_t>(), c.size(0),
232 - c.size(1));
396 + a_scales.data_ptr<float>(), nullptr, nullptr, nullptr,
397 + bias->data_ptr<scalar_t>(), c.size(0), c.size(1));
398 } else {
234 - dynamic_output_scale_impl<false>(
399 + // Compute C=s_a * C_inter
400 + dynamic_quant_epilogue<false, true, false, scalar_t>(
401 tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
236 - a_scales.data_ptr<float>(), (scalar_t*)(0), c.size(0), c.size(1));
402 + a_scales.data_ptr<float>(), nullptr, nullptr, nullptr, nullptr,
403 + c.size(0), c.size(1));
404 }
405 } else {
406 // per-tensor
407 if (bias.has_value()) {
408 + // Compute C=s_a * s_b * (A@B) + bias
409 DNNLPrimitiveHelper<false>::gemm_s8s8_jit(
410 a.data_ptr<int8_t>(), b.data_ptr<int8_t>(), c.data_ptr<scalar_t>(),
411 bias->data_ptr<scalar_t>(), a.size(0), b.size(1), a.size(1),
412 a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
413 a_scales.numel(), b_scales.numel());
414 } else {
247 - DNNLPrimitiveHelper<false>::gemm_s8s8_jit(
415 + // Compute C=s_a * s_b * (A@B)
416 + DNNLPrimitiveHelper<false>::gemm_s8s8_jit<scalar_t, void>(
417 a.data_ptr<int8_t>(), b.data_ptr<int8_t>(), c.data_ptr<scalar_t>(),
249 - (void*)(0), a.size(0), b.size(1), a.size(1),
418 + nullptr, a.size(0), b.size(1), a.size(1),
419 a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
420 a_scales.numel(), b_scales.numel());
421 }
@@ -254,6 +423,127 @@ void int8_scaled_mm(torch::Tensor& c, // [M, OC], row-major
423 });
424 }
425
426 +void int8_scaled_mm_azp(torch::Tensor& c, // [M, OC], row-major
427 + const torch::Tensor& a, // [M, IC], row-major
428 + const torch::Tensor& b, // [IC, OC], column-major
429 + const torch::Tensor& a_scales, // [1] or [M]
430 + const torch::Tensor& b_scales, // [1] or [OC]
431 + const torch::Tensor& azp_adj, // [OC]
432 + const c10::optional<torch::Tensor>& azp, // [1] or [M]
433 + const c10::optional<torch::Tensor>& bias // [OC]
434 +) {
435 + CPU_KERNEL_GUARD_IN(cutlass_scaled_mm_azp)
436 + // Checks for conformality
437 + TORCH_CHECK(a.dtype() == torch::kInt8 && b.dtype() == torch::kInt8,
438 + "int8_scaled_mm_azp only supports INT8 inputs.")
439 + TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
440 + TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
441 + b.size(1) == c.size(1));
442 + TORCH_CHECK(a_scales.numel() == 1 || a_scales.numel() == a.size(0));
443 + TORCH_CHECK(b_scales.numel() == 1 || b_scales.numel() == b.size(1));
444 +
445 + // Check for strides and alignment
446 + TORCH_CHECK(a.stride(1) == 1 && c.stride(1) == 1); // Row-major
447 + TORCH_CHECK(b.stride(0) == 1); // Column-major
448 + TORCH_CHECK(c.stride(0) % 16 == 0 &&
449 + b.stride(1) % 16 == 0); // 16 Byte Alignment
450 + TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
451 +
452 + if (bias) {
453 + TORCH_CHECK(bias->numel() == b.size(1) && bias->is_contiguous());
454 + }
455 + if (azp) {
456 + TORCH_CHECK(azp->numel() == a.size(0) && azp->is_contiguous());
457 + }
458 + TORCH_CHECK(azp_adj.numel() == b.size(1) && azp_adj.is_contiguous());
459 +
460 + // azp & bias types
461 + TORCH_CHECK(azp_adj.dtype() == torch::kInt32);
462 + TORCH_CHECK(!azp || azp->dtype() == torch::kInt32);
463 + TORCH_CHECK(!bias || bias->dtype() == c.dtype(),
464 + "currently bias dtype must match output dtype ", c.dtype());
465 +
466 + VLLM_DISPATCH_FLOATING_TYPES(c.scalar_type(), "int8_scaled_mm_azp", [&] {
467 + torch::Tensor tmp_fp32_out = torch::empty_like(c, ::at::ScalarType::Float);
468 + if (a_scales.numel() != 1) {
469 + // per-token
470 + // Note: oneDNN doesn't support per-token activation quantization
471 + // Compute C_inter=s_b * (A@B)
472 + DNNLPrimitiveHelper<true>::gemm_s8s8_jit<float, void>(
473 + a.data_ptr<int8_t>(), b.data_ptr<int8_t>(),
474 + tmp_fp32_out.data_ptr<float>(), nullptr, a.size(0), b.size(1),
475 + a.size(1), nullptr, b_scales.data_ptr<float>(), 0, b_scales.numel());
476 + if (bias.has_value()) {
477 + // Compute C=s_a * C_inter - s_a * s_b * azp * azp_adj + bias
478 + if (b_scales.numel() != 1) {
479 + // Per-Channel
480 + dynamic_quant_epilogue<true, true, true>(
481 + tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
482 + a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
483 + azp->data_ptr<int32_t>(), azp_adj.data_ptr<int32_t>(),
484 + bias->data_ptr<scalar_t>(), c.size(0), c.size(1));
485 + } else {
486 + // Per-Tensor
487 + dynamic_quant_epilogue<true, false, true>(
488 + tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
489 + a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
490 + azp->data_ptr<int32_t>(), azp_adj.data_ptr<int32_t>(),
491 + bias->data_ptr<scalar_t>(), c.size(0), c.size(1));
492 + }
493 + } else {
494 + // Compute C=s_a * C_inter - s_a * s_b * azp * azp_adj
495 + if (b_scales.numel() != 1) {
496 + // Per-Channel
497 + dynamic_quant_epilogue<true, true, false, scalar_t>(
498 + tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
499 + a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
500 + azp->data_ptr<int32_t>(), azp_adj.data_ptr<int32_t>(), nullptr,
501 + c.size(0), c.size(1));
502 + } else {
503 + // Per-Tensor
504 + dynamic_quant_epilogue<true, false, false, scalar_t>(
505 + tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
506 + a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
507 + azp->data_ptr<int32_t>(), azp_adj.data_ptr<int32_t>(), nullptr,
508 + c.size(0), c.size(1));
509 + }
510 + }
511 + } else {
512 + // per-tensor
513 + if (bias.has_value()) {
514 + // Compute C_inter=s_a * s_b * (A@B) + bias
515 + DNNLPrimitiveHelper<false>::gemm_s8s8_jit(
516 + a.data_ptr<int8_t>(), b.data_ptr<int8_t>(),
517 + tmp_fp32_out.data_ptr<float>(), bias->data_ptr<scalar_t>(),
518 + a.size(0), b.size(1), a.size(1), a_scales.data_ptr<float>(),
519 + b_scales.data_ptr<float>(), a_scales.numel(), b_scales.numel());
520 + } else {
521 + // Compute C_inter=s_a * s_b * (A@B)
522 + DNNLPrimitiveHelper<false>::gemm_s8s8_jit<float, void>(
523 + a.data_ptr<int8_t>(), b.data_ptr<int8_t>(),
524 + tmp_fp32_out.data_ptr<float>(), nullptr, a.size(0), b.size(1),
525 + a.size(1), a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
526 + a_scales.numel(), b_scales.numel());
527 + }
528 +
529 + // Compute C=C_inter - s_a * s_b * azp_adj
530 + if (b_scales.numel() != 1) {
531 + // Per-Channel
532 + static_quant_epilogue<true>(
533 + tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
534 + *a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
535 + azp_adj.data_ptr<int32_t>(), a.size(0), b.size(1));
536 + } else {
537 + // Per-Tensor
538 + static_quant_epilogue<false>(
539 + tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
540 + *a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
541 + azp_adj.data_ptr<int32_t>(), a.size(0), b.size(1));
542 + }
543 + }
544 + });
545 +}
546 +
547 // static-per-tensor quantization.
548 void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
549 const torch::Tensor& input, // [..., hidden_size]
@@ -263,15 +553,22 @@ void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
553 TORCH_CHECK(input.is_contiguous());
554 TORCH_CHECK(out.is_contiguous());
555 TORCH_CHECK(scale.numel() == 1);
266 - TORCH_CHECK(!azp.has_value(), "Zero point is not supported on CPU.");
556 + TORCH_CHECK(!azp.has_value() || azp->numel() == 1);
557
558 const int hidden_size = input.size(-1);
559 const int num_tokens = input.numel() / hidden_size;
560 VLLM_DISPATCH_FLOATING_TYPES(
561 input.scalar_type(), "static_scaled_int8_quant_impl", [&] {
272 - static_scaled_int8_quant_impl(
273 - input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
274 - scale.data_ptr<float>(), num_tokens, hidden_size);
562 + if (azp.has_value()) {
563 + static_scaled_int8_quant_impl<true>(
564 + input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
565 + scale.data_ptr<float>(), azp->data_ptr<int32_t>(), num_tokens,
566 + hidden_size);
567 + } else {
568 + static_scaled_int8_quant_impl<false>(
569 + input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
570 + scale.data_ptr<float>(), nullptr, num_tokens, hidden_size);
571 + }
572 });
573 }
574
@@ -284,14 +581,20 @@ void dynamic_scaled_int8_quant(
581 CPU_KERNEL_GUARD_IN(dynamic_scaled_int8_quant)
582 TORCH_CHECK(input.is_contiguous());
583 TORCH_CHECK(out.is_contiguous());
287 - TORCH_CHECK(!azp.has_value(), "Zero point is not supported on CPU.");
584
585 int const hidden_size = input.size(-1);
586 int const num_tokens = input.numel() / hidden_size;
587 VLLM_DISPATCH_FLOATING_TYPES(
588 input.scalar_type(), "dynamic_scaled_int8_quant_impl", [&] {
293 - dynamic_scaled_int8_quant_impl(
294 - input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
295 - scale.data_ptr<float>(), num_tokens, hidden_size);
589 + if (azp.has_value()) {
590 + dynamic_scaled_int8_quant_impl<true>(
591 + input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
592 + scale.data_ptr<float>(), azp->data_ptr<int32_t>(), num_tokens,
593 + hidden_size);
594 + } else {
595 + dynamic_scaled_int8_quant_impl<false>(
596 + input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
597 + scale.data_ptr<float>(), nullptr, num_tokens, hidden_size);
598 + }
599 });
600 }
csrc/cpu/torch_bindings.cpp
+15
@@ -11,6 +11,13 @@ void int8_scaled_mm(torch::Tensor& c, const torch::Tensor& a,
11 const torch::Tensor& b_scales,
12 const c10::optional<torch::Tensor>& bias);
13
14 +void int8_scaled_mm_azp(torch::Tensor& c, const torch::Tensor& a,
15 + const torch::Tensor& b, const torch::Tensor& a_scales,
16 + const torch::Tensor& b_scales,
17 + const torch::Tensor& azp_adj,
18 + const c10::optional<torch::Tensor>& azp,
19 + const c10::optional<torch::Tensor>& bias);
20 +
21 TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
22 // vLLM custom ops
23
@@ -111,6 +118,14 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
118 " Tensor b, Tensor a_scales,"
119 " Tensor b_scales, Tensor? bias) -> ()");
120 ops.impl("cutlass_scaled_mm", torch::kCPU, &int8_scaled_mm);
121 + // w8a8 GEMM, supporting asymmetric per-tensor or per-row/column
122 + // quantization.
123 + ops.def(
124 + "cutlass_scaled_mm_azp(Tensor! out, Tensor a,"
125 + " Tensor b, Tensor a_scales,"
126 + " Tensor b_scales, Tensor azp_adj,"
127 + " Tensor? azp, Tensor? bias) -> ()");
128 + ops.impl("cutlass_scaled_mm_azp", torch::kCPU, &int8_scaled_mm_azp);
129 #endif
130 }
131
docs/source/getting_started/cpu-installation.rst
-14
@@ -59,20 +59,6 @@ Build from source
59 $ pip install cmake>=3.26 wheel packaging ninja "setuptools-scm>=8" numpy
60 $ pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu
61
62 -- Third, build and install oneDNN library from source:
63 -
64 -.. code-block:: console
65 -
66 - $ git clone -b rls-v3.5 https://github.com/oneapi-src/oneDNN.git
67 - $ cmake -B ./oneDNN/build -S ./oneDNN -G Ninja -DONEDNN_LIBRARY_TYPE=STATIC \
68 - -DONEDNN_BUILD_DOC=OFF \
69 - -DONEDNN_BUILD_EXAMPLES=OFF \
70 - -DONEDNN_BUILD_TESTS=OFF \
71 - -DONEDNN_BUILD_GRAPH=OFF \
72 - -DONEDNN_ENABLE_WORKLOAD=INFERENCE \
73 - -DONEDNN_ENABLE_PRIMITIVE=MATMUL
74 - $ cmake --build ./oneDNN/build --target install --config Release
75 -
62 - Finally, build and install vLLM CPU backend:
63
64 .. code-block:: console