@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0b952af458
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Hardware][Intel] Support compressed-tensor W8A8 for CPU backend (#7257)

Li, Jiang committed Sep 12, 2024 at 00:46 UTC 0b952af458ce86a69e58333f956081ab4b2665de
18 files changed +686 -43
.buildkite/run-cpu-test.sh
+6
@@ -30,6 +30,12 @@ docker exec cpu-test bash -c "
30 --ignore=tests/models/test_jamba.py \
31 --ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
32
33 +# Run compressed-tensor test
34 +docker exec cpu-test bash -c "
35 + pytest -s -v \
36 + tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_static_setup \
37 + tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_dynanmic_per_token"
38 +
39 # online inference
40 docker exec cpu-test bash -c "
41 export VLLM_CPU_KVCACHE_SPACE=10
Dockerfile.cpu
+17 -1
@@ -2,6 +2,10 @@
2
3 FROM ubuntu:22.04 AS cpu-test-1
4
5 +ENV CCACHE_DIR=/root/.cache/ccache
6 +
7 +ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache
8 +
9 RUN --mount=type=cache,target=/var/cache/apt \
10 apt-get update -y \
11 && apt-get install -y curl ccache git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 libnuma-dev \
@@ -26,6 +30,19 @@ RUN --mount=type=cache,target=/root/.cache/pip \
30 pip install --upgrade pip && \
31 pip install -r requirements-build.txt
32
33 +# install oneDNN
34 +RUN git clone -b rls-v3.5 https://github.com/oneapi-src/oneDNN.git
35 +
36 +RUN --mount=type=cache,target=/root/.cache/ccache \
37 + cmake -B ./oneDNN/build -S ./oneDNN -G Ninja -DONEDNN_LIBRARY_TYPE=STATIC \
38 + -DONEDNN_BUILD_DOC=OFF \
39 + -DONEDNN_BUILD_EXAMPLES=OFF \
40 + -DONEDNN_BUILD_TESTS=OFF \
41 + -DONEDNN_BUILD_GRAPH=OFF \
42 + -DONEDNN_ENABLE_WORKLOAD=INFERENCE \
43 + -DONEDNN_ENABLE_PRIMITIVE=MATMUL && \
44 + cmake --build ./oneDNN/build --target install --config Release
45 +
46 FROM cpu-test-1 AS build
47
48 WORKDIR /workspace/vllm
@@ -41,7 +58,6 @@ COPY ./ ./
58 ARG VLLM_CPU_DISABLE_AVX512
59 ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512}
60
44 -ENV CCACHE_DIR=/root/.cache/ccache
61 RUN --mount=type=cache,target=/root/.cache/pip \
62 --mount=type=cache,target=/root/.cache/ccache \
63 VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel && \
cmake/cpu_extension.cmake
+12 -6
@@ -1,4 +1,5 @@
1 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
2 +set(CMAKE_CXX_STANDARD 17)
3
4 #
5 # Define environment variables for special configurations
@@ -83,12 +84,7 @@ endif()
84
85 message(STATUS "CPU extension compile flags: ${CXX_COMPILE_FLAGS}")
86
86 -list(APPEND LIBS "numa")
87 -
88 -
89 -#
90 -# Define extension targets
91 -#
87 +list(APPEND LIBS dnnl numa)
88
89 #
90 # _C extension
@@ -102,6 +98,16 @@ set(VLLM_EXT_SRC
98 "csrc/cpu/pos_encoding.cpp"
99 "csrc/cpu/torch_bindings.cpp")
100
101 +if (AVX512_FOUND AND NOT AVX512_DISABLED)
102 + set(VLLM_EXT_SRC
103 + "csrc/cpu/quant.cpp"
104 + ${VLLM_EXT_SRC})
105 +endif()
106 +
107 +#
108 +# Define extension targets
109 +#
110 +
111 define_gpu_extension_target(
112 _C
113 DESTINATION vllm
csrc/cpu/cpu_types_x86.hpp
+60 -2
@@ -24,8 +24,8 @@ namespace vec_op {
24 #define CPU_KERNEL_GUARD_OUT(NAME)
25 #else
26 #define CPU_KERNEL_GUARD_IN(NAME) \
27 - std::cout << #NAME << " invoked." << std::endl;
28 -#define CPU_KERNEL_GUARD_OUT(NAME) std::cout << #NAME << " exit." << std::endl;
27 + RECORD_FUNCTION(#NAME, c10::ArrayRef<c10::IValue>({}));
28 +#define CPU_KERNEL_GUARD_OUT(NAME)
29 #endif
30
31 #define FORCE_INLINE __attribute__((always_inline)) inline
@@ -106,6 +106,12 @@ struct BF16Vec16 : public Vec<BF16Vec16> {
106 explicit BF16Vec16(const FP32Vec16 &);
107
108 void save(void *ptr) const { *reinterpret_cast<__m256i *>(ptr) = reg; }
109 +
110 + void save(void* ptr, const int elem_num) const {
111 + constexpr uint32_t M = 0xFFFFFFFF;
112 + __mmask16 mask = _cvtu32_mask16(M >> (32 - elem_num));
113 + _mm256_mask_storeu_epi16(ptr, mask, reg);
114 + }
115 };
116
117 #ifdef __AVX512F__
@@ -313,8 +319,28 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
319 return FP32Vec16(_mm512_div_ps(reg, b.reg));
320 }
321
322 + FP32Vec16 clamp(const FP32Vec16& min, const FP32Vec16& max) const {
323 + return FP32Vec16(_mm512_min_ps(max.reg, _mm512_max_ps(min.reg, reg)));
324 + }
325 +
326 + FP32Vec16 max(const FP32Vec16& b) const {
327 + return FP32Vec16(_mm512_max_ps(reg, b.reg));
328 + }
329 +
330 + FP32Vec16 max(const FP32Vec16& b, const int elem_num) const {
331 + constexpr uint32_t M = 0xFFFFFFFF;
332 + __mmask16 mask = _cvtu32_mask16(M >> (32 - elem_num));
333 + return FP32Vec16(_mm512_mask_max_ps(reg, mask, reg, b.reg));
334 + }
335 +
336 + FP32Vec16 abs() const {
337 + return FP32Vec16(_mm512_abs_ps(reg));
338 + }
339 +
340 float reduce_sum() const { return _mm512_reduce_add_ps(reg); }
341
342 + float reduce_max() const { return _mm512_reduce_max_ps(reg); }
343 +
344 template <int group_size> float reduce_sub_sum(int idx) {
345 static_assert(VEC_ELEM_NUM % group_size == 0);
346 constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
@@ -323,6 +349,12 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
349 }
350
351 void save(float *ptr) const { _mm512_storeu_ps(ptr, reg); }
352 +
353 + void save(float* ptr, const int elem_num) const {
354 + constexpr uint32_t M = 0xFFFFFFFF;
355 + __mmask16 mask = _cvtu32_mask16(M >> (32 - elem_num));
356 + _mm512_mask_storeu_ps(ptr, mask, reg);
357 + }
358 };
359 #else
360 struct FP32Vec16 : public Vec<FP32Vec16> {
@@ -433,6 +465,32 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
465 };
466 #endif
467
468 +#ifdef __AVX512F__
469 +struct INT8Vec16: public Vec<INT8Vec16> {
470 + constexpr static int VEC_ELEM_NUM = 16;
471 + union AliasReg {
472 + __m128i reg;
473 + int8_t values[VEC_ELEM_NUM];
474 + };
475 +
476 + __m128i reg;
477 +
478 + explicit INT8Vec16(const FP32Vec16& vec) : reg(
479 + _mm512_cvtepi32_epi8(_mm512_cvt_roundps_epi32(vec.reg, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC))
480 + ) {}
481 +
482 + void save(int8_t* ptr) const {
483 + _mm_storeu_epi8(ptr, reg);
484 + }
485 +
486 + void save(int8_t* ptr, const int elem_num) const {
487 + constexpr uint32_t M = 0xFFFFFFFF;
488 + __mmask16 mask = _cvtu32_mask16(M >> (32 - elem_num));
489 + _mm_mask_storeu_epi8(ptr, mask, reg);
490 + }
491 +};
492 +#endif
493 +
494 template <typename T> struct VecType { using vec_type = void; };
495
496 template <typename T> using vec_t = typename VecType<T>::vec_type;
csrc/cpu/dnnl_helper.hpp new
+168
@@ -0,0 +1,168 @@
1 +#ifndef DNNL_HELPER_HPP
2 +#define DNNL_HELPER_HPP
3 +
4 +#include <c10/util/BFloat16.h>
5 +
6 +#include "oneapi/dnnl/dnnl.hpp"
7 +
8 +namespace {
9 +template <typename T>
10 +struct DNNLType {
11 + static constexpr dnnl::memory::data_type type =
12 + dnnl::memory::data_type::undef;
13 +};
14 +
15 +template <>
16 +struct DNNLType<int8_t> {
17 + static constexpr dnnl::memory::data_type type = dnnl::memory::data_type::s8;
18 +};
19 +
20 +template <>
21 +struct DNNLType<int32_t> {
22 + static constexpr dnnl::memory::data_type type = dnnl::memory::data_type::s32;
23 +};
24 +
25 +template <>
26 +struct DNNLType<float> {
27 + static constexpr dnnl::memory::data_type type = dnnl::memory::data_type::f32;
28 +};
29 +
30 +template <>
31 +struct DNNLType<c10::BFloat16> {
32 + static constexpr dnnl::memory::data_type type = dnnl::memory::data_type::bf16;
33 +};
34 +
35 +template <typename T>
36 +constexpr inline dnnl::memory::data_type get_dnnl_type() {
37 + return DNNLType<std::decay_t<T>>::type;
38 +}
39 +}; // namespace
40 +
41 +template <bool InputNoScale>
42 +class DNNLPrimitiveHelper {
43 + public:
44 + // I8 input GEMM kernel (C = a_scales * A @ (b_scales * B^T) + bias)
45 + // A: [M, K], row-major
46 + // B: [K, N], column-major
47 + // C: [M, N], row-major
48 + // bias: [N], row-major, optional
49 + // a_scales: [MS]
50 + // b_scales: [NS]
51 + // Note: Due to the limitation of oneDNN
52 + // (https://github.com/oneapi-src/oneDNN/issues/1636), the quantized bias is
53 + // not supported.
54 + template <typename OutputT, typename BiasT>
55 + static void gemm_s8s8_jit(const int8_t* a, const int8_t* b, OutputT* c,
56 + const BiasT* bias, dnnl_dim_t M, dnnl_dim_t N,
57 + dnnl_dim_t K, const float* a_scales,
58 + const float* b_scales, dnnl_dim_t MS,
59 + dnnl_dim_t NS) {
60 + auto&& OutputType = get_dnnl_type<OutputT>();
61 + auto&& BiasType = get_dnnl_type<BiasT>();
62 +
63 + dnnl::memory::desc a_md({M, K}, dnnl::memory::data_type::s8, {K, 1});
64 + dnnl::memory::desc b_md({K, N}, dnnl::memory::data_type::s8, {1, K});
65 + dnnl::memory::desc c_md({M, N}, OutputType, {N, 1});
66 +
67 + dnnl::primitive_attr attr;
68 + if constexpr (!InputNoScale) {
69 + if (MS == 1) {
70 + // per-tensor
71 + attr.set_scales_mask(DNNL_ARG_SRC, 0);
72 + } else {
73 + // per-token
74 + TORCH_CHECK(false, "per-token quantization is unsupported.");
75 + }
76 + }
77 +
78 + if (NS == 1) {
79 + // per-tensor
80 + attr.set_scales_mask(DNNL_ARG_WEIGHTS, 0);
81 + } else {
82 + // per-channel
83 + attr.set_scales_mask(DNNL_ARG_WEIGHTS, 2);
84 + }
85 +
86 + dnnl::matmul::primitive_desc matmul_pd;
87 + if (bias) {
88 + dnnl::memory::desc bias_md({1, N}, BiasType, {N, 1});
89 + matmul_pd = dnnl::matmul::primitive_desc(default_engine(), a_md, b_md,
90 + bias_md, c_md, attr);
91 + } else {
92 + matmul_pd = dnnl::matmul::primitive_desc(default_engine(), a_md, b_md,
93 + c_md, attr);
94 + }
95 + dnnl::matmul matmul(matmul_pd);
96 +
97 + auto& engine = default_engine();
98 +
99 + dnnl::memory a_m(a_md, engine, (void*)a);
100 + dnnl::memory b_m(b_md, engine, (void*)b);
101 + dnnl::memory c_m(c_md, engine, (void*)c);
102 + dnnl::memory a_scales_m({{MS}, dnnl::memory::data_type::f32, {1}}, engine,
103 + (void*)a_scales);
104 + dnnl::memory b_scales_m({{NS}, dnnl::memory::data_type::f32, {1}}, engine,
105 + (void*)b_scales);
106 +
107 + auto& stream = default_stream();
108 + if constexpr (InputNoScale) {
109 + if (bias) {
110 + dnnl::memory::desc bias_md({N}, BiasType, {1});
111 + dnnl::memory bias_m(bias_md, engine, (void*)bias);
112 + matmul.execute(
113 + stream, {
114 + {DNNL_ARG_SRC, a_m},
115 + {DNNL_ARG_WEIGHTS, b_m},
116 + {DNNL_ARG_BIAS, bias_m},
117 + {DNNL_ARG_DST, c_m},
118 + {DNNL_ARG_ATTR_SCALES | DNNL_ARG_WEIGHTS, b_scales_m},
119 + });
120 + } else {
121 + matmul.execute(
122 + stream, {
123 + {DNNL_ARG_SRC, a_m},
124 + {DNNL_ARG_WEIGHTS, b_m},
125 + {DNNL_ARG_DST, c_m},
126 + {DNNL_ARG_ATTR_SCALES | DNNL_ARG_WEIGHTS, b_scales_m},
127 + });
128 + }
129 + } else {
130 + if (bias) {
131 + dnnl::memory::desc bias_md({N}, BiasType, {1});
132 + dnnl::memory bias_m(bias_md, engine, (void*)bias);
133 + matmul.execute(
134 + stream, {
135 + {DNNL_ARG_SRC, a_m},
136 + {DNNL_ARG_WEIGHTS, b_m},
137 + {DNNL_ARG_BIAS, bias_m},
138 + {DNNL_ARG_DST, c_m},
139 + {DNNL_ARG_ATTR_SCALES | DNNL_ARG_SRC, a_scales_m},
140 + {DNNL_ARG_ATTR_SCALES | DNNL_ARG_WEIGHTS, b_scales_m},
141 + });
142 + } else {
143 + matmul.execute(
144 + stream, {
145 + {DNNL_ARG_SRC, a_m},
146 + {DNNL_ARG_WEIGHTS, b_m},
147 + {DNNL_ARG_DST, c_m},
148 + {DNNL_ARG_ATTR_SCALES | DNNL_ARG_SRC, a_scales_m},
149 + {DNNL_ARG_ATTR_SCALES | DNNL_ARG_WEIGHTS, b_scales_m},
150 + });
151 + }
152 + }
153 + stream.wait();
154 + }
155 +
156 + private:
157 + static dnnl::engine& default_engine() {
158 + static dnnl::engine engine(dnnl::engine::kind::cpu, 0);
159 + return engine;
160 + }
161 +
162 + static dnnl::stream& default_stream() {
163 + static dnnl::stream stream(default_engine());
164 + return stream;
165 + }
166 +};
167 +
168 +#endif
csrc/cpu/quant.cpp new
+294
@@ -0,0 +1,294 @@
1 +#include "cpu_types.hpp"
2 +#include "dnnl_helper.hpp"
3 +
4 +namespace {
5 +template <typename scalar_t>
6 +struct KernelVecType {
7 + using load_vec_type = void;
8 + using cvt_vec_type = void;
9 +};
10 +
11 +template <>
12 +struct KernelVecType<float> {
13 + using load_vec_type = vec_op::FP32Vec16;
14 + using cvt_vec_type = vec_op::FP32Vec16;
15 +};
16 +
17 +template <>
18 +struct KernelVecType<c10::BFloat16> {
19 + using load_vec_type = vec_op::BF16Vec16;
20 + using cvt_vec_type = vec_op::FP32Vec16;
21 +};
22 +
23 +#ifdef __AVX512F__
24 +template <typename scalar_t>
25 +void static_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
26 + const float* scale, const int num_tokens,
27 + const int hidden_size) {
28 + using load_vec_t = typename KernelVecType<scalar_t>::load_vec_type;
29 + using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
30 + constexpr int vec_elem_num = load_vec_t::VEC_ELEM_NUM;
31 +
32 + constexpr float i8_min =
33 + static_cast<float>(std::numeric_limits<int8_t>::min());
34 + constexpr float i8_max =
35 + static_cast<float>(std::numeric_limits<int8_t>::max());
36 + const cvt_vec_t inv_scale(1.0 / *scale);
37 + const cvt_vec_t i8_min_vec(i8_min);
38 + const cvt_vec_t i8_max_vec(i8_max);
39 +
40 + #pragma omp parallel for
41 + for (int i = 0; i < num_tokens; ++i) {
42 + int j = 0;
43 + for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
44 + load_vec_t elems(input + i * hidden_size + j);
45 + cvt_vec_t elems_fp32(elems);
46 + elems_fp32 = (elems_fp32 * inv_scale).clamp(i8_min_vec, i8_max_vec);
47 + vec_op::INT8Vec16 elems_int8(elems_fp32);
48 + elems_int8.save(output + i * hidden_size + j);
49 + }
50 +
51 + load_vec_t elems(input + i * hidden_size + j);
52 + cvt_vec_t elems_fp32(elems);
53 + elems_fp32 = (elems_fp32 * inv_scale).clamp(i8_min_vec, i8_max_vec);
54 + vec_op::INT8Vec16 elems_int8(elems_fp32);
55 +
56 + if (j + vec_elem_num == hidden_size) {
57 + elems_int8.save(output + i * hidden_size + j);
58 + } else {
59 + elems_int8.save(output + i * hidden_size + j, hidden_size - j);
60 + }
61 + }
62 +}
63 +
64 +template <typename scalar_t>
65 +void dynamic_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
66 + float* scale, const int num_tokens,
67 + const int hidden_size) {
68 + using load_vec_t = typename KernelVecType<scalar_t>::load_vec_type;
69 + using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
70 + constexpr int vec_elem_num = load_vec_t::VEC_ELEM_NUM;
71 +
72 + #pragma omp parallel for
73 + for (int i = 0; i < num_tokens; ++i) {
74 + cvt_vec_t max_abs(0.0);
75 + {
76 + int j = 0;
77 + for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
78 + load_vec_t elems(input + i * hidden_size + j);
79 + cvt_vec_t elems_fp32(elems);
80 + max_abs = max_abs.max(elems_fp32.abs());
81 + }
82 +
83 + load_vec_t elems(input + i * hidden_size + j);
84 + cvt_vec_t elems_fp32(elems);
85 +
86 + if (j + vec_elem_num == hidden_size) {
87 + max_abs = max_abs.max(elems_fp32.abs());
88 + } else {
89 + max_abs = max_abs.max(elems_fp32.abs(), hidden_size - j);
90 + }
91 + }
92 +
93 + float scale_val = max_abs.reduce_max() / 127.0f;
94 + scale[i] = scale_val;
95 + const cvt_vec_t inv_scale(1.0 / scale_val);
96 +
97 + {
98 + int j = 0;
99 + for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
100 + load_vec_t elems(input + i * hidden_size + j);
101 + cvt_vec_t elems_fp32(elems);
102 + elems_fp32 = (elems_fp32 * inv_scale);
103 + vec_op::INT8Vec16 elems_int8(elems_fp32);
104 + elems_int8.save(output + i * hidden_size + j);
105 + }
106 +
107 + load_vec_t elems(input + i * hidden_size + j);
108 + cvt_vec_t elems_fp32(elems);
109 + elems_fp32 = (elems_fp32 * inv_scale);
110 + vec_op::INT8Vec16 elems_int8(elems_fp32);
111 +
112 + if (j + vec_elem_num == hidden_size) {
113 + elems_int8.save(output + i * hidden_size + j);
114 + } else {
115 + elems_int8.save(output + i * hidden_size + j, hidden_size - j);
116 + }
117 + }
118 + }
119 +}
120 +
121 +template <bool Bias, typename scalar_t>
122 +void dynamic_output_scale_impl(const float* input, scalar_t* output,
123 + const float* scale, const scalar_t* bias,
124 + const int num_tokens, const int hidden_size) {
125 + CPU_KERNEL_GUARD_IN(dynamic_output_scale_impl)
126 + using load_vec_t = typename KernelVecType<scalar_t>::load_vec_type;
127 + using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
128 + constexpr int vec_elem_num = load_vec_t::VEC_ELEM_NUM;
129 +
130 + #pragma omp parallel for
131 + for (int i = 0; i < num_tokens; ++i) {
132 + int j = 0;
133 + cvt_vec_t token_scale_vec(scale[i]);
134 + for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
135 + cvt_vec_t elems_fp32(input + i * hidden_size + j);
136 + elems_fp32 = elems_fp32 * token_scale_vec;
137 +
138 + if constexpr (Bias) {
139 + load_vec_t bias_vec(bias + j);
140 + cvt_vec_t bias_vec_fp32(bias_vec);
141 + elems_fp32 = elems_fp32 + bias_vec_fp32;
142 + }
143 +
144 + load_vec_t elems_out(elems_fp32);
145 + elems_out.save(output + i * hidden_size + j);
146 + }
147 +
148 + cvt_vec_t elems_fp32(input + i * hidden_size + j);
149 + elems_fp32 = elems_fp32 * token_scale_vec;
150 +
151 + if constexpr (Bias) {
152 + load_vec_t bias_vec(bias + j);
153 + cvt_vec_t bias_vec_fp32(bias_vec);
154 + elems_fp32 = elems_fp32 + bias_vec_fp32;
155 + }
156 +
157 + load_vec_t elems_out(elems_fp32);
158 +
159 + if (j + vec_elem_num == hidden_size) {
160 + elems_out.save(output + i * hidden_size + j);
161 + } else {
162 + elems_out.save(output + i * hidden_size + j, hidden_size - j);
163 + }
164 + }
165 +}
166 +#else
167 +template <typename scalar_t>
168 +void static_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
169 + const float* scale, const int num_tokens,
170 + const int hidden_size) {
171 + TORCH_CHECK(false, "static_scaled_int8_quant_impl requires AVX512 support.")
172 +}
173 +
174 +template <typename scalar_t>
175 +void dynamic_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
176 + float* scale, const int num_tokens,
177 + const int hidden_size) {
178 + TORCH_CHECK(false, "dynamic_scaled_int8_quant_impl requires AVX512 support.")
179 +}
180 +
181 +template <typename scalar_t>
182 +void dynamic_output_scale_impl() {
183 + TORCH_CHECK(false, "dynamic_output_scale_impl requires AVX512 support.")
184 +}
185 +#endif
186 +} // namespace
187 +
188 +void int8_scaled_mm(torch::Tensor& c, // [M, OC], row-major
189 + const torch::Tensor& a, // [M, IC], row-major
190 + const torch::Tensor& b, // [IC, OC], column-major
191 + const torch::Tensor& a_scales, // [1] or [M]
192 + const torch::Tensor& b_scales, // [1] or [OC]
193 + const c10::optional<torch::Tensor>& bias // [OC]
194 +) {
195 + CPU_KERNEL_GUARD_IN(cutlass_scaled_mm)
196 + // Checks for conformality
197 + TORCH_CHECK(a.dtype() == torch::kInt8 && b.dtype() == torch::kInt8,
198 + "int8_scaled_mm only supports INT8 inputs.")
199 + TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
200 + TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
201 + b.size(1) == c.size(1));
202 + TORCH_CHECK(a_scales.numel() == 1 || a_scales.numel() == a.size(0));
203 + TORCH_CHECK(b_scales.numel() == 1 || b_scales.numel() == b.size(1));
204 +
205 + // Check for strides and alignment
206 + TORCH_CHECK(a.stride(1) == 1 && c.stride(1) == 1); // Row-major
207 + TORCH_CHECK(b.stride(0) == 1); // Column-major
208 + TORCH_CHECK(c.stride(0) % 16 == 0 &&
209 + b.stride(1) % 16 == 0); // 16 Byte Alignment
210 + TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
211 +
212 + if (bias) {
213 + TORCH_CHECK(bias->numel() == b.size(1) && bias->is_contiguous() &&
214 + bias->dim() == 1);
215 + }
216 +
217 + VLLM_DISPATCH_FLOATING_TYPES(c.scalar_type(), "cutlass_scaled_mm", [&] {
218 + if (a_scales.numel() != 1) {
219 + // per-token
220 + // Note: oneDNN doesn't support per-token activation quantization
221 + torch::Tensor tmp_fp32_out =
222 + torch::empty_like(c, ::at::ScalarType::Float);
223 + DNNLPrimitiveHelper<true>::gemm_s8s8_jit(
224 + a.data_ptr<int8_t>(), b.data_ptr<int8_t>(),
225 + tmp_fp32_out.data_ptr<float>(), (void*)(0), a.size(0), b.size(1),
226 + a.size(1), (float*)(0), b_scales.data_ptr<float>(), 0,
227 + b_scales.numel());
228 + if (bias.has_value()) {
229 + dynamic_output_scale_impl<true>(
230 + tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
231 + a_scales.data_ptr<float>(), bias->data_ptr<scalar_t>(), c.size(0),
232 + c.size(1));
233 + } else {
234 + dynamic_output_scale_impl<false>(
235 + tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
236 + a_scales.data_ptr<float>(), (scalar_t*)(0), c.size(0), c.size(1));
237 + }
238 + } else {
239 + // per-tensor
240 + if (bias.has_value()) {
241 + DNNLPrimitiveHelper<false>::gemm_s8s8_jit(
242 + a.data_ptr<int8_t>(), b.data_ptr<int8_t>(), c.data_ptr<scalar_t>(),
243 + bias->data_ptr<scalar_t>(), a.size(0), b.size(1), a.size(1),
244 + a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
245 + a_scales.numel(), b_scales.numel());
246 + } else {
247 + DNNLPrimitiveHelper<false>::gemm_s8s8_jit(
248 + a.data_ptr<int8_t>(), b.data_ptr<int8_t>(), c.data_ptr<scalar_t>(),
249 + (void*)(0), a.size(0), b.size(1), a.size(1),
250 + a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
251 + a_scales.numel(), b_scales.numel());
252 + }
253 + }
254 + });
255 +}
256 +
257 +// static-per-tensor quantization.
258 +void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
259 + const torch::Tensor& input, // [..., hidden_size]
260 + const torch::Tensor& scale) {
261 + CPU_KERNEL_GUARD_IN(static_scaled_int8_quant)
262 + TORCH_CHECK(input.is_contiguous());
263 + TORCH_CHECK(out.is_contiguous());
264 + TORCH_CHECK(scale.numel() == 1);
265 +
266 + const int hidden_size = input.size(-1);
267 + const int num_tokens = input.numel() / hidden_size;
268 + VLLM_DISPATCH_FLOATING_TYPES(
269 + input.scalar_type(), "static_scaled_int8_quant_impl", [&] {
270 + static_scaled_int8_quant_impl(
271 + input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
272 + scale.data_ptr<float>(), num_tokens, hidden_size);
273 + });
274 +}
275 +
276 +// dynamic-per-token quantization.
277 +void dynamic_scaled_int8_quant(
278 + torch::Tensor& out, // [..., hidden_size]
279 + const torch::Tensor& input, // [..., hidden_size]
280 + torch::Tensor& scale // [..., 1]
281 +) {
282 + CPU_KERNEL_GUARD_IN(dynamic_scaled_int8_quant)
283 + TORCH_CHECK(input.is_contiguous());
284 + TORCH_CHECK(out.is_contiguous());
285 +
286 + int const hidden_size = input.size(-1);
287 + int const num_tokens = input.numel() / hidden_size;
288 + VLLM_DISPATCH_FLOATING_TYPES(
289 + input.scalar_type(), "dynamic_scaled_int8_quant_impl", [&] {
290 + dynamic_scaled_int8_quant_impl(
291 + input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
292 + scale.data_ptr<float>(), num_tokens, hidden_size);
293 + });
294 +}
csrc/cpu/torch_bindings.cpp
+29 -2
@@ -4,7 +4,12 @@
4
5 #include <torch/library.h>
6
7 -void init_cpu_threads_env(const std::string& cpu_ids);
7 +std::string init_cpu_threads_env(const std::string& cpu_ids);
8 +
9 +void int8_scaled_mm(torch::Tensor& c, const torch::Tensor& a,
10 + const torch::Tensor& b, const torch::Tensor& a_scales,
11 + const torch::Tensor& b_scales,
12 + const c10::optional<torch::Tensor>& bias);
13
14 TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
15 // vLLM custom ops
@@ -84,6 +89,28 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
89 " Tensor! key, int head_size,"
90 " Tensor cos_sin_cache, bool is_neox) -> ()");
91 ops.impl("rotary_embedding", torch::kCPU, &rotary_embedding);
92 +
93 + // Quantization
94 +#ifdef __AVX512F__
95 + // Compute int8 quantized tensor for given scaling factor.
96 + ops.def(
97 + "static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale) -> "
98 + "()");
99 + ops.impl("static_scaled_int8_quant", torch::kCPU, &static_scaled_int8_quant);
100 + // Compute int8 quantized tensor and scaling factor
101 + ops.def(
102 + "dynamic_scaled_int8_quant(Tensor! out, Tensor input, Tensor! scale) -> "
103 + "()");
104 + ops.impl("dynamic_scaled_int8_quant", torch::kCPU,
105 + &dynamic_scaled_int8_quant);
106 + // W8A8 GEMM, supporting symmetric per-tensor or per-row/column
107 + // quantization.
108 + ops.def(
109 + "cutlass_scaled_mm(Tensor! out, Tensor a,"
110 + " Tensor b, Tensor a_scales,"
111 + " Tensor b_scales, Tensor? bias) -> ()");
112 + ops.impl("cutlass_scaled_mm", torch::kCPU, &int8_scaled_mm);
113 +#endif
114 }
115
116 TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
@@ -111,7 +138,7 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
138
139 TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _utils), utils) {
140 // CPU utils
114 - utils.def("init_cpu_threads_env(str cpu_ids) -> ()", &init_cpu_threads_env);
141 + utils.def("init_cpu_threads_env(str cpu_ids) -> str", &init_cpu_threads_env);
142 }
143
144 REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/cpu/utils.cpp
+32 -7
@@ -5,7 +5,7 @@
5
6 #include "cpu_types.hpp"
7
8 -void init_cpu_threads_env(const std::string& cpu_ids) {
8 +std::string init_cpu_threads_env(const std::string& cpu_ids) {
9 bitmask* omp_cpu_mask = numa_parse_cpustring(cpu_ids.c_str());
10 TORCH_CHECK(omp_cpu_mask->size > 0);
11 std::vector<int> omp_cpu_ids;
@@ -51,15 +51,40 @@ void init_cpu_threads_env(const std::string& cpu_ids) {
51 torch::set_num_threads((int)omp_cpu_ids.size());
52 TORCH_CHECK_EQ(omp_cpu_ids.size(), torch::get_num_threads());
53 TORCH_CHECK_EQ(omp_cpu_ids.size(), omp_get_max_threads());
54 +
55 + std::vector<std::pair<int, int>> thread_core_mapping;
56 + thread_core_mapping.reserve(omp_cpu_ids.size());
57 + omp_lock_t writelock;
58 + omp_init_lock(&writelock);
59 +
60 #pragma omp parallel for schedule(static, 1)
61 for (size_t i = 0; i < omp_cpu_ids.size(); ++i) {
56 - cpu_set_t* mask = CPU_ALLOC(omp_cpu_mask->size);
57 - size_t size = CPU_ALLOC_SIZE(omp_cpu_mask->size);
58 - CPU_ZERO_S(size, mask);
59 - CPU_SET_S(omp_cpu_ids[i], size, mask);
60 - sched_setaffinity(0, sizeof(cpu_set_t), mask);
61 - CPU_FREE(mask);
62 + cpu_set_t mask;
63 + CPU_ZERO(&mask);
64 + CPU_SET(omp_cpu_ids[i], &mask);
65 + int ret = sched_setaffinity(0, sizeof(cpu_set_t), &mask);
66 + if (ret == -1) {
67 + TORCH_CHECK(false,
68 + "sched_setaffinity failed. errno: " + std::to_string(errno));
69 + }
70 +
71 + omp_set_lock(&writelock);
72 + thread_core_mapping.emplace_back(gettid(), omp_cpu_ids[i]);
73 + omp_unset_lock(&writelock);
74 }
75
76 + omp_destroy_lock(&writelock);
77 +
78 numa_free_nodemask(omp_cpu_mask);
79 +
80 + std::stringstream ss;
81 + ss << "OMP threads binding of Process " << getpid() << ":\n";
82 + std::sort(thread_core_mapping.begin(), thread_core_mapping.end(),
83 + [](auto&& a, auto&& b) { return a.second < b.second; });
84 + for (auto&& item : thread_core_mapping) {
85 + ss << "\t"
86 + << "OMP tid: " << item.first << ", core " << item.second << "\n";
87 + }
88 +
89 + return ss.str();
90 }
tests/quantization/test_compressed_tensors.py
+2 -2
@@ -56,7 +56,7 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
56 assert qkv_proj.weight_scale.dtype is torch.float32
57 assert qkv_proj.input_scale.dtype is torch.float32
58
59 - output = llm.generate_greedy("Hello my name is", max_tokens=20)
59 + output = llm.generate_greedy(["Hello my name is"], max_tokens=20)
60 assert output
61
62
@@ -85,7 +85,7 @@ def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner, model_args):
85 assert qkv_proj.scheme.strategy == strategy
86 assert qkv_proj.weight.dtype is torch.int8
87
88 - output = llm.generate_greedy("Hello my name is", max_tokens=20)
88 + output = llm.generate_greedy(["Hello my name is"], max_tokens=20)
89 assert output
90
91
vllm/config.py
+2 -1
@@ -876,7 +876,8 @@ class ParallelConfig:
876 from vllm.executor import ray_utils
877 backend = "mp"
878 ray_found = ray_utils.ray_is_available()
879 - if cuda_device_count_stateless() < self.world_size:
879 + if (torch.cuda.is_available()
880 + and cuda_device_count_stateless() < self.world_size):
881 if not ray_found:
882 raise ValueError("Unable to load Ray which is "
883 "required for multi-node inference, "
vllm/executor/cpu_executor.py
+14 -1
@@ -5,7 +5,8 @@ from typing import Any, Awaitable, List, Optional, Set, Tuple, Union
5 import torch
6
7 import vllm.envs as envs
8 -from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
8 +from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9 + SchedulerConfig)
10 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
11 from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
12 ResultHandler, WorkerMonitor)
@@ -60,6 +61,8 @@ class CPUExecutor(ExecutorBase):
61 self.cache_config = _verify_and_get_cache_config(self.cache_config)
62 self.scheduler_config = _verify_and_get_scheduler_config(
63 self.scheduler_config)
64 + self.parallel_config = _verify_and_get_parallel_config(
65 + self.parallel_config)
66
67 # Multiprocessing-based executor does not support multi-node setting.
68 # Since it only works for single node, we can use the loopback address
@@ -359,6 +362,16 @@ def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
362 return config
363
364
365 +def _verify_and_get_parallel_config(config: ParallelConfig) -> ParallelConfig:
366 + if (config.distributed_executor_backend is not None
367 + and config.distributed_executor_backend != "mp"):
368 + logger.warning(
369 + "%s is not supported on CPU, fallback to mp distributed executor "
370 + "backend.", config.distributed_executor_backend)
371 + config.distributed_executor_backend = "mp"
372 + return config
373 +
374 +
375 def _driver_method_invoker(driver, method: str, *args, **kwargs):
376 return getattr(driver, method)(*args, **kwargs)
377
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+13 -9
@@ -116,15 +116,19 @@ class CompressedTensorsConfig(QuantizationConfig):
116 def _check_scheme_supported(self,
117 min_capability: int,
118 error: bool = True) -> bool:
119 - capability = current_platform.get_device_capability()
120 - capability = capability[0] * 10 + capability[1]
121 - supported = capability >= min_capability
122 - if error and not supported:
123 - raise RuntimeError(
124 - "Quantization scheme is not supported for ",
125 - f"the current GPU. Min capability: {min_capability}. ",
126 - f"Current capability: {capability}.")
127 - return supported
119 + capability = current_platform.get_device_capability() # type: ignore
120 +
121 + if capability is not None:
122 + capability = capability[0] * 10 + capability[1]
123 + supported = capability >= min_capability
124 + if error and not supported:
125 + raise RuntimeError(
126 + "Quantization scheme is not supported for ",
127 + f"the current GPU. Min capability: {min_capability}. ",
128 + f"Current capability: {capability}.")
129 + return supported
130 + else:
131 + return False
132
133 def _is_static_tensor_w8a8(self, weight_quant: BaseModel,
134 input_quant: BaseModel) -> bool:
vllm/model_executor/model_loader/loader.py
+3 -2
@@ -95,8 +95,9 @@ def _get_quantization_config(
95 """Get the quantization config."""
96 if model_config.quantization is not None:
97 quant_config = get_quant_config(model_config, load_config)
98 - if not current_platform.is_tpu():
99 - capability = current_platform.get_device_capability()
98 + capability = current_platform.get_device_capability() # type: ignore
99 +
100 + if capability is not None:
101 capability = capability[0] * 10 + capability[1]
102 if capability < quant_config.get_min_capability():
103 raise ValueError(
vllm/platforms/__init__.py
+10
@@ -42,6 +42,13 @@ try:
42 except Exception:
43 pass
44
45 +is_cpu = False
46 +try:
47 + from importlib.metadata import version
48 + is_cpu = "cpu" in version("vllm")
49 +except Exception:
50 + pass
51 +
52 if is_tpu:
53 # people might install pytorch built with cuda but run on tpu
54 # so we need to check tpu first
@@ -53,6 +60,9 @@ elif is_cuda:
60 elif is_rocm:
61 from .rocm import RocmPlatform
62 current_platform = RocmPlatform()
63 +elif is_cpu:
64 + from .cpu import CpuPlatform
65 + current_platform = CpuPlatform()
66 else:
67 current_platform = UnspecifiedPlatform()
68
vllm/platforms/cpu.py new
+15
@@ -0,0 +1,15 @@
1 +import torch
2 +
3 +from .interface import Platform, PlatformEnum
4 +
5 +
6 +class CpuPlatform(Platform):
7 + _enum = PlatformEnum.CPU
8 +
9 + @staticmethod
10 + def get_device_name(device_id: int = 0) -> str:
11 + return "cpu"
12 +
13 + @staticmethod
14 + def inference_mode():
15 + return torch.no_grad()
vllm/platforms/interface.py
+7 -3
@@ -1,5 +1,5 @@
1 import enum
2 -from typing import Tuple
2 +from typing import Optional, Tuple
3
4 import torch
5
@@ -8,6 +8,7 @@ class PlatformEnum(enum.Enum):
8 CUDA = enum.auto()
9 ROCM = enum.auto()
10 TPU = enum.auto()
11 + CPU = enum.auto()
12 UNSPECIFIED = enum.auto()
13
14
@@ -23,9 +24,12 @@ class Platform:
24 def is_tpu(self) -> bool:
25 return self._enum == PlatformEnum.TPU
26
27 + def is_cpu(self) -> bool:
28 + return self._enum == PlatformEnum.CPU
29 +
30 @staticmethod
27 - def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
28 - raise NotImplementedError
31 + def get_device_capability(device_id: int = 0) -> Optional[Tuple[int, int]]:
32 + return None
33
34 @staticmethod
35 def get_device_name(device_id: int = 0) -> str:
vllm/platforms/tpu.py
-6
@@ -1,5 +1,3 @@
1 -from typing import Tuple
2 -
1 import torch
2
3 from .interface import Platform, PlatformEnum
@@ -8,10 +6,6 @@ from .interface import Platform, PlatformEnum
6 class TpuPlatform(Platform):
7 _enum = PlatformEnum.TPU
8
11 - @staticmethod
12 - def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
13 - raise RuntimeError("TPU does not have device capability.")
14 -
9 @staticmethod
10 def inference_mode():
11 return torch.no_grad()
vllm/worker/cpu_worker.py
+2 -1
@@ -207,7 +207,8 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
207
208 def init_device(self) -> None:
209 if self.local_omp_cpuid != "all":
210 - torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)
210 + ret = torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)
211 + logger.info(ret)
212
213 self.init_distributed_environment()
214 # Set random seed.