A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Hardware][Intel] Support compressed-tensor W8A8 for CPU backend (#7257)
Li, Jiang committed
Sep 12, 2024 at 00:46 UTC
0b952af458ce86a69e58333f956081ab4b2665de
18 files changed
+686
-43
.buildkite/run-cpu-test.sh
+6
@@ -30,6 +30,12 @@ docker exec cpu-test bash -c "
30
--ignore=tests/models/test_jamba.py \
31
--ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
32
33
+# Run compressed-tensor test
34
+docker exec cpu-test bash -c "
35
+ pytest -s -v \
36
+ tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_static_setup \
37
+ tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_dynanmic_per_token"
38
+
39
# online inference
40
docker exec cpu-test bash -c "
41
export VLLM_CPU_KVCACHE_SPACE=10
Dockerfile.cpu
+17
-1
@@ -2,6 +2,10 @@
2
3
FROM ubuntu:22.04 AS cpu-test-1
4
5
+ENV CCACHE_DIR=/root/.cache/ccache
6
+
7
+ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache
8
+
9
RUN --mount=type=cache,target=/var/cache/apt \
10
apt-get update -y \
11
&& apt-get install -y curl ccache git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 libnuma-dev \
@@ -26,6 +30,19 @@ RUN --mount=type=cache,target=/root/.cache/pip \
30
pip install --upgrade pip && \
31
pip install -r requirements-build.txt
32
33
+# install oneDNN
34
+RUN git clone -b rls-v3.5 https://github.com/oneapi-src/oneDNN.git
35
+
36
+RUN --mount=type=cache,target=/root/.cache/ccache \
37
+ cmake -B ./oneDNN/build -S ./oneDNN -G Ninja -DONEDNN_LIBRARY_TYPE=STATIC \
38
+ -DONEDNN_BUILD_DOC=OFF \
39
+ -DONEDNN_BUILD_EXAMPLES=OFF \
40
+ -DONEDNN_BUILD_TESTS=OFF \
41
+ -DONEDNN_BUILD_GRAPH=OFF \
42
+ -DONEDNN_ENABLE_WORKLOAD=INFERENCE \
43
+ -DONEDNN_ENABLE_PRIMITIVE=MATMUL && \
44
+ cmake --build ./oneDNN/build --target install --config Release
45
+
46
FROM cpu-test-1 AS build
47
48
WORKDIR /workspace/vllm
@@ -41,7 +58,6 @@ COPY ./ ./
58
ARG VLLM_CPU_DISABLE_AVX512
59
ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512}
60
44
-ENV CCACHE_DIR=/root/.cache/ccache
61
RUN --mount=type=cache,target=/root/.cache/pip \
62
--mount=type=cache,target=/root/.cache/ccache \
63
VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel && \
cmake/cpu_extension.cmake
+12
-6
@@ -1,4 +1,5 @@
1
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
2
+set(CMAKE_CXX_STANDARD 17)
3
4
#
5
# Define environment variables for special configurations
@@ -83,12 +84,7 @@ endif()
84
85
message(STATUS "CPU extension compile flags: ${CXX_COMPILE_FLAGS}")
86
86
-list(APPEND LIBS "numa")
87
-
88
-
89
-#
90
-# Define extension targets
91
-#
87
+list(APPEND LIBS dnnl numa)
88
89
#
90
# _C extension
@@ -102,6 +98,16 @@ set(VLLM_EXT_SRC
98
"csrc/cpu/pos_encoding.cpp"
99
"csrc/cpu/torch_bindings.cpp")
100
101
+if (AVX512_FOUND AND NOT AVX512_DISABLED)
102
+ set(VLLM_EXT_SRC
103
+ "csrc/cpu/quant.cpp"
104
+ ${VLLM_EXT_SRC})
105
+endif()
106
+
107
+#
108
+# Define extension targets
109
+#
110
+
111
define_gpu_extension_target(
112
_C
113
DESTINATION vllm
csrc/cpu/cpu_types_x86.hpp
+60
-2
@@ -24,8 +24,8 @@ namespace vec_op {
24
#define CPU_KERNEL_GUARD_OUT(NAME)
25
#else
26
#define CPU_KERNEL_GUARD_IN(NAME) \
27
- std::cout << #NAME << " invoked." << std::endl;
28
-#define CPU_KERNEL_GUARD_OUT(NAME) std::cout << #NAME << " exit." << std::endl;
27
+ RECORD_FUNCTION(#NAME, c10::ArrayRef<c10::IValue>({}));
28
+#define CPU_KERNEL_GUARD_OUT(NAME)
29
#endif
30
31
#define FORCE_INLINE __attribute__((always_inline)) inline
@@ -106,6 +106,12 @@ struct BF16Vec16 : public Vec<BF16Vec16> {
106
explicit BF16Vec16(const FP32Vec16 &);
107
108
void save(void *ptr) const { *reinterpret_cast<__m256i *>(ptr) = reg; }
109
+
110
+ void save(void* ptr, const int elem_num) const {
111
+ constexpr uint32_t M = 0xFFFFFFFF;
112
+ __mmask16 mask = _cvtu32_mask16(M >> (32 - elem_num));
113
+ _mm256_mask_storeu_epi16(ptr, mask, reg);
114
+ }
115
};
116
117
#ifdef __AVX512F__
@@ -313,8 +319,28 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
319
return FP32Vec16(_mm512_div_ps(reg, b.reg));
320
}
321
322
+ FP32Vec16 clamp(const FP32Vec16& min, const FP32Vec16& max) const {
323
+ return FP32Vec16(_mm512_min_ps(max.reg, _mm512_max_ps(min.reg, reg)));
324
+ }
325
+
326
+ FP32Vec16 max(const FP32Vec16& b) const {
327
+ return FP32Vec16(_mm512_max_ps(reg, b.reg));
328
+ }
329
+
330
+ FP32Vec16 max(const FP32Vec16& b, const int elem_num) const {
331
+ constexpr uint32_t M = 0xFFFFFFFF;
332
+ __mmask16 mask = _cvtu32_mask16(M >> (32 - elem_num));
333
+ return FP32Vec16(_mm512_mask_max_ps(reg, mask, reg, b.reg));
334
+ }
335
+
336
+ FP32Vec16 abs() const {
337
+ return FP32Vec16(_mm512_abs_ps(reg));
338
+ }
339
+
340
float reduce_sum() const { return _mm512_reduce_add_ps(reg); }
341
342
+ float reduce_max() const { return _mm512_reduce_max_ps(reg); }
343
+
344
template <int group_size> float reduce_sub_sum(int idx) {
345
static_assert(VEC_ELEM_NUM % group_size == 0);
346
constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
@@ -323,6 +349,12 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
349
}
350
351
void save(float *ptr) const { _mm512_storeu_ps(ptr, reg); }
352
+
353
+ void save(float* ptr, const int elem_num) const {
354
+ constexpr uint32_t M = 0xFFFFFFFF;
355
+ __mmask16 mask = _cvtu32_mask16(M >> (32 - elem_num));
356
+ _mm512_mask_storeu_ps(ptr, mask, reg);
357
+ }
358
};
359
#else
360
struct FP32Vec16 : public Vec<FP32Vec16> {
@@ -433,6 +465,32 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
465
};
466
#endif
467
468
+#ifdef __AVX512F__
469
+struct INT8Vec16: public Vec<INT8Vec16> {
470
+ constexpr static int VEC_ELEM_NUM = 16;
471
+ union AliasReg {
472
+ __m128i reg;
473
+ int8_t values[VEC_ELEM_NUM];
474
+ };
475
+
476
+ __m128i reg;
477
+
478
+ explicit INT8Vec16(const FP32Vec16& vec) : reg(
479
+ _mm512_cvtepi32_epi8(_mm512_cvt_roundps_epi32(vec.reg, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC))
480
+ ) {}
481
+
482
+ void save(int8_t* ptr) const {
483
+ _mm_storeu_epi8(ptr, reg);
484
+ }
485
+
486
+ void save(int8_t* ptr, const int elem_num) const {
487
+ constexpr uint32_t M = 0xFFFFFFFF;
488
+ __mmask16 mask = _cvtu32_mask16(M >> (32 - elem_num));
489
+ _mm_mask_storeu_epi8(ptr, mask, reg);
490
+ }
491
+};
492
+#endif
493
+
494
template <typename T> struct VecType { using vec_type = void; };
495
496
template <typename T> using vec_t = typename VecType<T>::vec_type;
csrc/cpu/dnnl_helper.hpp
new
+168
@@ -0,0 +1,168 @@
1
+#ifndef DNNL_HELPER_HPP
2
+#define DNNL_HELPER_HPP
3
+
4
+#include <c10/util/BFloat16.h>
5
+
6
+#include "oneapi/dnnl/dnnl.hpp"
7
+
8
+namespace {
9
+template <typename T>
10
+struct DNNLType {
11
+ static constexpr dnnl::memory::data_type type =
12
+ dnnl::memory::data_type::undef;
13
+};
14
+
15
+template <>
16
+struct DNNLType<int8_t> {
17
+ static constexpr dnnl::memory::data_type type = dnnl::memory::data_type::s8;
18
+};
19
+
20
+template <>
21
+struct DNNLType<int32_t> {
22
+ static constexpr dnnl::memory::data_type type = dnnl::memory::data_type::s32;
23
+};
24
+
25
+template <>
26
+struct DNNLType<float> {
27
+ static constexpr dnnl::memory::data_type type = dnnl::memory::data_type::f32;
28
+};
29
+
30
+template <>
31
+struct DNNLType<c10::BFloat16> {
32
+ static constexpr dnnl::memory::data_type type = dnnl::memory::data_type::bf16;
33
+};
34
+
35
+template <typename T>
36
+constexpr inline dnnl::memory::data_type get_dnnl_type() {
37
+ return DNNLType<std::decay_t<T>>::type;
38
+}
39
+}; // namespace
40
+
41
+template <bool InputNoScale>
42
+class DNNLPrimitiveHelper {
43
+ public:
44
+ // I8 input GEMM kernel (C = a_scales * A @ (b_scales * B^T) + bias)
45
+ // A: [M, K], row-major
46
+ // B: [K, N], column-major
47
+ // C: [M, N], row-major
48
+ // bias: [N], row-major, optional
49
+ // a_scales: [MS]
50
+ // b_scales: [NS]
51
+ // Note: Due to the limitation of oneDNN
52
+ // (https://github.com/oneapi-src/oneDNN/issues/1636), the quantized bias is
53
+ // not supported.
54
+ template <typename OutputT, typename BiasT>
55
+ static void gemm_s8s8_jit(const int8_t* a, const int8_t* b, OutputT* c,
56
+ const BiasT* bias, dnnl_dim_t M, dnnl_dim_t N,
57
+ dnnl_dim_t K, const float* a_scales,
58
+ const float* b_scales, dnnl_dim_t MS,
59
+ dnnl_dim_t NS) {
60
+ auto&& OutputType = get_dnnl_type<OutputT>();
61
+ auto&& BiasType = get_dnnl_type<BiasT>();
62
+
63
+ dnnl::memory::desc a_md({M, K}, dnnl::memory::data_type::s8, {K, 1});
64
+ dnnl::memory::desc b_md({K, N}, dnnl::memory::data_type::s8, {1, K});
65
+ dnnl::memory::desc c_md({M, N}, OutputType, {N, 1});
66
+
67
+ dnnl::primitive_attr attr;
68
+ if constexpr (!InputNoScale) {
69
+ if (MS == 1) {
70
+ // per-tensor
71
+ attr.set_scales_mask(DNNL_ARG_SRC, 0);
72
+ } else {
73
+ // per-token
74
+ TORCH_CHECK(false, "per-token quantization is unsupported.");
75
+ }
76
+ }
77
+
78
+ if (NS == 1) {
79
+ // per-tensor
80
+ attr.set_scales_mask(DNNL_ARG_WEIGHTS, 0);
81
+ } else {
82
+ // per-channel
83
+ attr.set_scales_mask(DNNL_ARG_WEIGHTS, 2);
84
+ }
85
+
86
+ dnnl::matmul::primitive_desc matmul_pd;
87
+ if (bias) {
88
+ dnnl::memory::desc bias_md({1, N}, BiasType, {N, 1});
89
+ matmul_pd = dnnl::matmul::primitive_desc(default_engine(), a_md, b_md,
90
+ bias_md, c_md, attr);
91
+ } else {
92
+ matmul_pd = dnnl::matmul::primitive_desc(default_engine(), a_md, b_md,
93
+ c_md, attr);
94
+ }
95
+ dnnl::matmul matmul(matmul_pd);
96
+
97
+ auto& engine = default_engine();
98
+
99
+ dnnl::memory a_m(a_md, engine, (void*)a);
100
+ dnnl::memory b_m(b_md, engine, (void*)b);
101
+ dnnl::memory c_m(c_md, engine, (void*)c);
102
+ dnnl::memory a_scales_m({{MS}, dnnl::memory::data_type::f32, {1}}, engine,
103
+ (void*)a_scales);
104
+ dnnl::memory b_scales_m({{NS}, dnnl::memory::data_type::f32, {1}}, engine,
105
+ (void*)b_scales);
106
+
107
+ auto& stream = default_stream();
108
+ if constexpr (InputNoScale) {
109
+ if (bias) {
110
+ dnnl::memory::desc bias_md({N}, BiasType, {1});
111
+ dnnl::memory bias_m(bias_md, engine, (void*)bias);
112
+ matmul.execute(
113
+ stream, {
114
+ {DNNL_ARG_SRC, a_m},
115
+ {DNNL_ARG_WEIGHTS, b_m},
116
+ {DNNL_ARG_BIAS, bias_m},
117
+ {DNNL_ARG_DST, c_m},
118
+ {DNNL_ARG_ATTR_SCALES | DNNL_ARG_WEIGHTS, b_scales_m},
119
+ });
120
+ } else {
121
+ matmul.execute(
122
+ stream, {
123
+ {DNNL_ARG_SRC, a_m},
124
+ {DNNL_ARG_WEIGHTS, b_m},
125
+ {DNNL_ARG_DST, c_m},
126
+ {DNNL_ARG_ATTR_SCALES | DNNL_ARG_WEIGHTS, b_scales_m},
127
+ });
128
+ }
129
+ } else {
130
+ if (bias) {
131
+ dnnl::memory::desc bias_md({N}, BiasType, {1});
132
+ dnnl::memory bias_m(bias_md, engine, (void*)bias);
133
+ matmul.execute(
134
+ stream, {
135
+ {DNNL_ARG_SRC, a_m},
136
+ {DNNL_ARG_WEIGHTS, b_m},
137
+ {DNNL_ARG_BIAS, bias_m},
138
+ {DNNL_ARG_DST, c_m},
139
+ {DNNL_ARG_ATTR_SCALES | DNNL_ARG_SRC, a_scales_m},
140
+ {DNNL_ARG_ATTR_SCALES | DNNL_ARG_WEIGHTS, b_scales_m},
141
+ });
142
+ } else {
143
+ matmul.execute(
144
+ stream, {
145
+ {DNNL_ARG_SRC, a_m},
146
+ {DNNL_ARG_WEIGHTS, b_m},
147
+ {DNNL_ARG_DST, c_m},
148
+ {DNNL_ARG_ATTR_SCALES | DNNL_ARG_SRC, a_scales_m},
149
+ {DNNL_ARG_ATTR_SCALES | DNNL_ARG_WEIGHTS, b_scales_m},
150
+ });
151
+ }
152
+ }
153
+ stream.wait();
154
+ }
155
+
156
+ private:
157
+ static dnnl::engine& default_engine() {
158
+ static dnnl::engine engine(dnnl::engine::kind::cpu, 0);
159
+ return engine;
160
+ }
161
+
162
+ static dnnl::stream& default_stream() {
163
+ static dnnl::stream stream(default_engine());
164
+ return stream;
165
+ }
166
+};
167
+
168
+#endif
csrc/cpu/quant.cpp
new
+294
@@ -0,0 +1,294 @@
1
+#include "cpu_types.hpp"
2
+#include "dnnl_helper.hpp"
3
+
4
+namespace {
5
+template <typename scalar_t>
6
+struct KernelVecType {
7
+ using load_vec_type = void;
8
+ using cvt_vec_type = void;
9
+};
10
+
11
+template <>
12
+struct KernelVecType<float> {
13
+ using load_vec_type = vec_op::FP32Vec16;
14
+ using cvt_vec_type = vec_op::FP32Vec16;
15
+};
16
+
17
+template <>
18
+struct KernelVecType<c10::BFloat16> {
19
+ using load_vec_type = vec_op::BF16Vec16;
20
+ using cvt_vec_type = vec_op::FP32Vec16;
21
+};
22
+
23
+#ifdef __AVX512F__
24
+template <typename scalar_t>
25
+void static_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
26
+ const float* scale, const int num_tokens,
27
+ const int hidden_size) {
28
+ using load_vec_t = typename KernelVecType<scalar_t>::load_vec_type;
29
+ using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
30
+ constexpr int vec_elem_num = load_vec_t::VEC_ELEM_NUM;
31
+
32
+ constexpr float i8_min =
33
+ static_cast<float>(std::numeric_limits<int8_t>::min());
34
+ constexpr float i8_max =
35
+ static_cast<float>(std::numeric_limits<int8_t>::max());
36
+ const cvt_vec_t inv_scale(1.0 / *scale);
37
+ const cvt_vec_t i8_min_vec(i8_min);
38
+ const cvt_vec_t i8_max_vec(i8_max);
39
+
40
+ #pragma omp parallel for
41
+ for (int i = 0; i < num_tokens; ++i) {
42
+ int j = 0;
43
+ for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
44
+ load_vec_t elems(input + i * hidden_size + j);
45
+ cvt_vec_t elems_fp32(elems);
46
+ elems_fp32 = (elems_fp32 * inv_scale).clamp(i8_min_vec, i8_max_vec);
47
+ vec_op::INT8Vec16 elems_int8(elems_fp32);
48
+ elems_int8.save(output + i * hidden_size + j);
49
+ }
50
+
51
+ load_vec_t elems(input + i * hidden_size + j);
52
+ cvt_vec_t elems_fp32(elems);
53
+ elems_fp32 = (elems_fp32 * inv_scale).clamp(i8_min_vec, i8_max_vec);
54
+ vec_op::INT8Vec16 elems_int8(elems_fp32);
55
+
56
+ if (j + vec_elem_num == hidden_size) {
57
+ elems_int8.save(output + i * hidden_size + j);
58
+ } else {
59
+ elems_int8.save(output + i * hidden_size + j, hidden_size - j);
60
+ }
61
+ }
62
+}
63
+
64
+template <typename scalar_t>
65
+void dynamic_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
66
+ float* scale, const int num_tokens,
67
+ const int hidden_size) {
68
+ using load_vec_t = typename KernelVecType<scalar_t>::load_vec_type;
69
+ using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
70
+ constexpr int vec_elem_num = load_vec_t::VEC_ELEM_NUM;
71
+
72
+ #pragma omp parallel for
73
+ for (int i = 0; i < num_tokens; ++i) {
74
+ cvt_vec_t max_abs(0.0);
75
+ {
76
+ int j = 0;
77
+ for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
78
+ load_vec_t elems(input + i * hidden_size + j);
79
+ cvt_vec_t elems_fp32(elems);
80
+ max_abs = max_abs.max(elems_fp32.abs());
81
+ }
82
+
83
+ load_vec_t elems(input + i * hidden_size + j);
84
+ cvt_vec_t elems_fp32(elems);
85
+
86
+ if (j + vec_elem_num == hidden_size) {
87
+ max_abs = max_abs.max(elems_fp32.abs());
88
+ } else {
89
+ max_abs = max_abs.max(elems_fp32.abs(), hidden_size - j);
90
+ }
91
+ }
92
+
93
+ float scale_val = max_abs.reduce_max() / 127.0f;
94
+ scale[i] = scale_val;
95
+ const cvt_vec_t inv_scale(1.0 / scale_val);
96
+
97
+ {
98
+ int j = 0;
99
+ for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
100
+ load_vec_t elems(input + i * hidden_size + j);
101
+ cvt_vec_t elems_fp32(elems);
102
+ elems_fp32 = (elems_fp32 * inv_scale);
103
+ vec_op::INT8Vec16 elems_int8(elems_fp32);
104
+ elems_int8.save(output + i * hidden_size + j);
105
+ }
106
+
107
+ load_vec_t elems(input + i * hidden_size + j);
108
+ cvt_vec_t elems_fp32(elems);
109
+ elems_fp32 = (elems_fp32 * inv_scale);
110
+ vec_op::INT8Vec16 elems_int8(elems_fp32);
111
+
112
+ if (j + vec_elem_num == hidden_size) {
113
+ elems_int8.save(output + i * hidden_size + j);
114
+ } else {
115
+ elems_int8.save(output + i * hidden_size + j, hidden_size - j);
116
+ }
117
+ }
118
+ }
119
+}
120
+
121
+template <bool Bias, typename scalar_t>
122
+void dynamic_output_scale_impl(const float* input, scalar_t* output,
123
+ const float* scale, const scalar_t* bias,
124
+ const int num_tokens, const int hidden_size) {
125
+ CPU_KERNEL_GUARD_IN(dynamic_output_scale_impl)
126
+ using load_vec_t = typename KernelVecType<scalar_t>::load_vec_type;
127
+ using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
128
+ constexpr int vec_elem_num = load_vec_t::VEC_ELEM_NUM;
129
+
130
+ #pragma omp parallel for
131
+ for (int i = 0; i < num_tokens; ++i) {
132
+ int j = 0;
133
+ cvt_vec_t token_scale_vec(scale[i]);
134
+ for (; j < hidden_size - vec_elem_num; j += vec_elem_num) {
135
+ cvt_vec_t elems_fp32(input + i * hidden_size + j);
136
+ elems_fp32 = elems_fp32 * token_scale_vec;
137
+
138
+ if constexpr (Bias) {
139
+ load_vec_t bias_vec(bias + j);
140
+ cvt_vec_t bias_vec_fp32(bias_vec);
141
+ elems_fp32 = elems_fp32 + bias_vec_fp32;
142
+ }
143
+
144
+ load_vec_t elems_out(elems_fp32);
145
+ elems_out.save(output + i * hidden_size + j);
146
+ }
147
+
148
+ cvt_vec_t elems_fp32(input + i * hidden_size + j);
149
+ elems_fp32 = elems_fp32 * token_scale_vec;
150
+
151
+ if constexpr (Bias) {
152
+ load_vec_t bias_vec(bias + j);
153
+ cvt_vec_t bias_vec_fp32(bias_vec);
154
+ elems_fp32 = elems_fp32 + bias_vec_fp32;
155
+ }
156
+
157
+ load_vec_t elems_out(elems_fp32);
158
+
159
+ if (j + vec_elem_num == hidden_size) {
160
+ elems_out.save(output + i * hidden_size + j);
161
+ } else {
162
+ elems_out.save(output + i * hidden_size + j, hidden_size - j);
163
+ }
164
+ }
165
+}
166
+#else
167
+template <typename scalar_t>
168
+void static_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
169
+ const float* scale, const int num_tokens,
170
+ const int hidden_size) {
171
+ TORCH_CHECK(false, "static_scaled_int8_quant_impl requires AVX512 support.")
172
+}
173
+
174
+template <typename scalar_t>
175
+void dynamic_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
176
+ float* scale, const int num_tokens,
177
+ const int hidden_size) {
178
+ TORCH_CHECK(false, "dynamic_scaled_int8_quant_impl requires AVX512 support.")
179
+}
180
+
181
+template <typename scalar_t>
182
+void dynamic_output_scale_impl() {
183
+ TORCH_CHECK(false, "dynamic_output_scale_impl requires AVX512 support.")
184
+}
185
+#endif
186
+} // namespace
187
+
188
+void int8_scaled_mm(torch::Tensor& c, // [M, OC], row-major
189
+ const torch::Tensor& a, // [M, IC], row-major
190
+ const torch::Tensor& b, // [IC, OC], column-major
191
+ const torch::Tensor& a_scales, // [1] or [M]
192
+ const torch::Tensor& b_scales, // [1] or [OC]
193
+ const c10::optional<torch::Tensor>& bias // [OC]
194
+) {
195
+ CPU_KERNEL_GUARD_IN(cutlass_scaled_mm)
196
+ // Checks for conformality
197
+ TORCH_CHECK(a.dtype() == torch::kInt8 && b.dtype() == torch::kInt8,
198
+ "int8_scaled_mm only supports INT8 inputs.")
199
+ TORCH_CHECK(a.dim() == 2 && b.dim() == 2 && c.dim() == 2);
200
+ TORCH_CHECK(c.size(0) == a.size(0) && a.size(1) == b.size(0) &&
201
+ b.size(1) == c.size(1));
202
+ TORCH_CHECK(a_scales.numel() == 1 || a_scales.numel() == a.size(0));
203
+ TORCH_CHECK(b_scales.numel() == 1 || b_scales.numel() == b.size(1));
204
+
205
+ // Check for strides and alignment
206
+ TORCH_CHECK(a.stride(1) == 1 && c.stride(1) == 1); // Row-major
207
+ TORCH_CHECK(b.stride(0) == 1); // Column-major
208
+ TORCH_CHECK(c.stride(0) % 16 == 0 &&
209
+ b.stride(1) % 16 == 0); // 16 Byte Alignment
210
+ TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
211
+
212
+ if (bias) {
213
+ TORCH_CHECK(bias->numel() == b.size(1) && bias->is_contiguous() &&
214
+ bias->dim() == 1);
215
+ }
216
+
217
+ VLLM_DISPATCH_FLOATING_TYPES(c.scalar_type(), "cutlass_scaled_mm", [&] {
218
+ if (a_scales.numel() != 1) {
219
+ // per-token
220
+ // Note: oneDNN doesn't support per-token activation quantization
221
+ torch::Tensor tmp_fp32_out =
222
+ torch::empty_like(c, ::at::ScalarType::Float);
223
+ DNNLPrimitiveHelper<true>::gemm_s8s8_jit(
224
+ a.data_ptr<int8_t>(), b.data_ptr<int8_t>(),
225
+ tmp_fp32_out.data_ptr<float>(), (void*)(0), a.size(0), b.size(1),
226
+ a.size(1), (float*)(0), b_scales.data_ptr<float>(), 0,
227
+ b_scales.numel());
228
+ if (bias.has_value()) {
229
+ dynamic_output_scale_impl<true>(
230
+ tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
231
+ a_scales.data_ptr<float>(), bias->data_ptr<scalar_t>(), c.size(0),
232
+ c.size(1));
233
+ } else {
234
+ dynamic_output_scale_impl<false>(
235
+ tmp_fp32_out.data_ptr<float>(), c.data_ptr<scalar_t>(),
236
+ a_scales.data_ptr<float>(), (scalar_t*)(0), c.size(0), c.size(1));
237
+ }
238
+ } else {
239
+ // per-tensor
240
+ if (bias.has_value()) {
241
+ DNNLPrimitiveHelper<false>::gemm_s8s8_jit(
242
+ a.data_ptr<int8_t>(), b.data_ptr<int8_t>(), c.data_ptr<scalar_t>(),
243
+ bias->data_ptr<scalar_t>(), a.size(0), b.size(1), a.size(1),
244
+ a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
245
+ a_scales.numel(), b_scales.numel());
246
+ } else {
247
+ DNNLPrimitiveHelper<false>::gemm_s8s8_jit(
248
+ a.data_ptr<int8_t>(), b.data_ptr<int8_t>(), c.data_ptr<scalar_t>(),
249
+ (void*)(0), a.size(0), b.size(1), a.size(1),
250
+ a_scales.data_ptr<float>(), b_scales.data_ptr<float>(),
251
+ a_scales.numel(), b_scales.numel());
252
+ }
253
+ }
254
+ });
255
+}
256
+
257
+// static-per-tensor quantization.
258
+void static_scaled_int8_quant(torch::Tensor& out, // [..., hidden_size]
259
+ const torch::Tensor& input, // [..., hidden_size]
260
+ const torch::Tensor& scale) {
261
+ CPU_KERNEL_GUARD_IN(static_scaled_int8_quant)
262
+ TORCH_CHECK(input.is_contiguous());
263
+ TORCH_CHECK(out.is_contiguous());
264
+ TORCH_CHECK(scale.numel() == 1);
265
+
266
+ const int hidden_size = input.size(-1);
267
+ const int num_tokens = input.numel() / hidden_size;
268
+ VLLM_DISPATCH_FLOATING_TYPES(
269
+ input.scalar_type(), "static_scaled_int8_quant_impl", [&] {
270
+ static_scaled_int8_quant_impl(
271
+ input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
272
+ scale.data_ptr<float>(), num_tokens, hidden_size);
273
+ });
274
+}
275
+
276
+// dynamic-per-token quantization.
277
+void dynamic_scaled_int8_quant(
278
+ torch::Tensor& out, // [..., hidden_size]
279
+ const torch::Tensor& input, // [..., hidden_size]
280
+ torch::Tensor& scale // [..., 1]
281
+) {
282
+ CPU_KERNEL_GUARD_IN(dynamic_scaled_int8_quant)
283
+ TORCH_CHECK(input.is_contiguous());
284
+ TORCH_CHECK(out.is_contiguous());
285
+
286
+ int const hidden_size = input.size(-1);
287
+ int const num_tokens = input.numel() / hidden_size;
288
+ VLLM_DISPATCH_FLOATING_TYPES(
289
+ input.scalar_type(), "dynamic_scaled_int8_quant_impl", [&] {
290
+ dynamic_scaled_int8_quant_impl(
291
+ input.data_ptr<scalar_t>(), out.data_ptr<int8_t>(),
292
+ scale.data_ptr<float>(), num_tokens, hidden_size);
293
+ });
294
+}
csrc/cpu/torch_bindings.cpp
+29
-2
@@ -4,7 +4,12 @@
4
5
#include <torch/library.h>
6
7
-void init_cpu_threads_env(const std::string& cpu_ids);
7
+std::string init_cpu_threads_env(const std::string& cpu_ids);
8
+
9
+void int8_scaled_mm(torch::Tensor& c, const torch::Tensor& a,
10
+ const torch::Tensor& b, const torch::Tensor& a_scales,
11
+ const torch::Tensor& b_scales,
12
+ const c10::optional<torch::Tensor>& bias);
13
14
TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
15
// vLLM custom ops
@@ -84,6 +89,28 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
89
" Tensor! key, int head_size,"
90
" Tensor cos_sin_cache, bool is_neox) -> ()");
91
ops.impl("rotary_embedding", torch::kCPU, &rotary_embedding);
92
+
93
+ // Quantization
94
+#ifdef __AVX512F__
95
+ // Compute int8 quantized tensor for given scaling factor.
96
+ ops.def(
97
+ "static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale) -> "
98
+ "()");
99
+ ops.impl("static_scaled_int8_quant", torch::kCPU, &static_scaled_int8_quant);
100
+ // Compute int8 quantized tensor and scaling factor
101
+ ops.def(
102
+ "dynamic_scaled_int8_quant(Tensor! out, Tensor input, Tensor! scale) -> "
103
+ "()");
104
+ ops.impl("dynamic_scaled_int8_quant", torch::kCPU,
105
+ &dynamic_scaled_int8_quant);
106
+ // W8A8 GEMM, supporting symmetric per-tensor or per-row/column
107
+ // quantization.
108
+ ops.def(
109
+ "cutlass_scaled_mm(Tensor! out, Tensor a,"
110
+ " Tensor b, Tensor a_scales,"
111
+ " Tensor b_scales, Tensor? bias) -> ()");
112
+ ops.impl("cutlass_scaled_mm", torch::kCPU, &int8_scaled_mm);
113
+#endif
114
}
115
116
TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
@@ -111,7 +138,7 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
138
139
TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _utils), utils) {
140
// CPU utils
114
- utils.def("init_cpu_threads_env(str cpu_ids) -> ()", &init_cpu_threads_env);
141
+ utils.def("init_cpu_threads_env(str cpu_ids) -> str", &init_cpu_threads_env);
142
}
143
144
REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/cpu/utils.cpp
+32
-7
@@ -5,7 +5,7 @@
5
6
#include "cpu_types.hpp"
7
8
-void init_cpu_threads_env(const std::string& cpu_ids) {
8
+std::string init_cpu_threads_env(const std::string& cpu_ids) {
9
bitmask* omp_cpu_mask = numa_parse_cpustring(cpu_ids.c_str());
10
TORCH_CHECK(omp_cpu_mask->size > 0);
11
std::vector<int> omp_cpu_ids;
@@ -51,15 +51,40 @@ void init_cpu_threads_env(const std::string& cpu_ids) {
51
torch::set_num_threads((int)omp_cpu_ids.size());
52
TORCH_CHECK_EQ(omp_cpu_ids.size(), torch::get_num_threads());
53
TORCH_CHECK_EQ(omp_cpu_ids.size(), omp_get_max_threads());
54
+
55
+ std::vector<std::pair<int, int>> thread_core_mapping;
56
+ thread_core_mapping.reserve(omp_cpu_ids.size());
57
+ omp_lock_t writelock;
58
+ omp_init_lock(&writelock);
59
+
60
#pragma omp parallel for schedule(static, 1)
61
for (size_t i = 0; i < omp_cpu_ids.size(); ++i) {
56
- cpu_set_t* mask = CPU_ALLOC(omp_cpu_mask->size);
57
- size_t size = CPU_ALLOC_SIZE(omp_cpu_mask->size);
58
- CPU_ZERO_S(size, mask);
59
- CPU_SET_S(omp_cpu_ids[i], size, mask);
60
- sched_setaffinity(0, sizeof(cpu_set_t), mask);
61
- CPU_FREE(mask);
62
+ cpu_set_t mask;
63
+ CPU_ZERO(&mask);
64
+ CPU_SET(omp_cpu_ids[i], &mask);
65
+ int ret = sched_setaffinity(0, sizeof(cpu_set_t), &mask);
66
+ if (ret == -1) {
67
+ TORCH_CHECK(false,
68
+ "sched_setaffinity failed. errno: " + std::to_string(errno));
69
+ }
70
+
71
+ omp_set_lock(&writelock);
72
+ thread_core_mapping.emplace_back(gettid(), omp_cpu_ids[i]);
73
+ omp_unset_lock(&writelock);
74
}
75
76
+ omp_destroy_lock(&writelock);
77
+
78
numa_free_nodemask(omp_cpu_mask);
79
+
80
+ std::stringstream ss;
81
+ ss << "OMP threads binding of Process " << getpid() << ":\n";
82
+ std::sort(thread_core_mapping.begin(), thread_core_mapping.end(),
83
+ [](auto&& a, auto&& b) { return a.second < b.second; });
84
+ for (auto&& item : thread_core_mapping) {
85
+ ss << "\t"
86
+ << "OMP tid: " << item.first << ", core " << item.second << "\n";
87
+ }
88
+
89
+ return ss.str();
90
}
tests/quantization/test_compressed_tensors.py
+2
-2
@@ -56,7 +56,7 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
56
assert qkv_proj.weight_scale.dtype is torch.float32
57
assert qkv_proj.input_scale.dtype is torch.float32
58
59
- output = llm.generate_greedy("Hello my name is", max_tokens=20)
59
+ output = llm.generate_greedy(["Hello my name is"], max_tokens=20)
60
assert output
61
62
@@ -85,7 +85,7 @@ def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner, model_args):
85
assert qkv_proj.scheme.strategy == strategy
86
assert qkv_proj.weight.dtype is torch.int8
87
88
- output = llm.generate_greedy("Hello my name is", max_tokens=20)
88
+ output = llm.generate_greedy(["Hello my name is"], max_tokens=20)
89
assert output
90
91
vllm/config.py
+2
-1
@@ -876,7 +876,8 @@ class ParallelConfig:
876
from vllm.executor import ray_utils
877
backend = "mp"
878
ray_found = ray_utils.ray_is_available()
879
- if cuda_device_count_stateless() < self.world_size:
879
+ if (torch.cuda.is_available()
880
+ and cuda_device_count_stateless() < self.world_size):
881
if not ray_found:
882
raise ValueError("Unable to load Ray which is "
883
"required for multi-node inference, "
vllm/executor/cpu_executor.py
+14
-1
@@ -5,7 +5,8 @@ from typing import Any, Awaitable, List, Optional, Set, Tuple, Union
5
import torch
6
7
import vllm.envs as envs
8
-from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
8
+from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9
+ SchedulerConfig)
10
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
11
from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
12
ResultHandler, WorkerMonitor)
@@ -60,6 +61,8 @@ class CPUExecutor(ExecutorBase):
61
self.cache_config = _verify_and_get_cache_config(self.cache_config)
62
self.scheduler_config = _verify_and_get_scheduler_config(
63
self.scheduler_config)
64
+ self.parallel_config = _verify_and_get_parallel_config(
65
+ self.parallel_config)
66
67
# Multiprocessing-based executor does not support multi-node setting.
68
# Since it only works for single node, we can use the loopback address
@@ -359,6 +362,16 @@ def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
362
return config
363
364
365
+def _verify_and_get_parallel_config(config: ParallelConfig) -> ParallelConfig:
366
+ if (config.distributed_executor_backend is not None
367
+ and config.distributed_executor_backend != "mp"):
368
+ logger.warning(
369
+ "%s is not supported on CPU, fallback to mp distributed executor "
370
+ "backend.", config.distributed_executor_backend)
371
+ config.distributed_executor_backend = "mp"
372
+ return config
373
+
374
+
375
def _driver_method_invoker(driver, method: str, *args, **kwargs):
376
return getattr(driver, method)(*args, **kwargs)
377
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+13
-9
@@ -116,15 +116,19 @@ class CompressedTensorsConfig(QuantizationConfig):
116
def _check_scheme_supported(self,
117
min_capability: int,
118
error: bool = True) -> bool:
119
- capability = current_platform.get_device_capability()
120
- capability = capability[0] * 10 + capability[1]
121
- supported = capability >= min_capability
122
- if error and not supported:
123
- raise RuntimeError(
124
- "Quantization scheme is not supported for ",
125
- f"the current GPU. Min capability: {min_capability}. ",
126
- f"Current capability: {capability}.")
127
- return supported
119
+ capability = current_platform.get_device_capability() # type: ignore
120
+
121
+ if capability is not None:
122
+ capability = capability[0] * 10 + capability[1]
123
+ supported = capability >= min_capability
124
+ if error and not supported:
125
+ raise RuntimeError(
126
+ "Quantization scheme is not supported for ",
127
+ f"the current GPU. Min capability: {min_capability}. ",
128
+ f"Current capability: {capability}.")
129
+ return supported
130
+ else:
131
+ return False
132
133
def _is_static_tensor_w8a8(self, weight_quant: BaseModel,
134
input_quant: BaseModel) -> bool:
vllm/model_executor/model_loader/loader.py
+3
-2
@@ -95,8 +95,9 @@ def _get_quantization_config(
95
"""Get the quantization config."""
96
if model_config.quantization is not None:
97
quant_config = get_quant_config(model_config, load_config)
98
- if not current_platform.is_tpu():
99
- capability = current_platform.get_device_capability()
98
+ capability = current_platform.get_device_capability() # type: ignore
99
+
100
+ if capability is not None:
101
capability = capability[0] * 10 + capability[1]
102
if capability < quant_config.get_min_capability():
103
raise ValueError(
vllm/platforms/__init__.py
+10
@@ -42,6 +42,13 @@ try:
42
except Exception:
43
pass
44
45
+is_cpu = False
46
+try:
47
+ from importlib.metadata import version
48
+ is_cpu = "cpu" in version("vllm")
49
+except Exception:
50
+ pass
51
+
52
if is_tpu:
53
# people might install pytorch built with cuda but run on tpu
54
# so we need to check tpu first
@@ -53,6 +60,9 @@ elif is_cuda:
60
elif is_rocm:
61
from .rocm import RocmPlatform
62
current_platform = RocmPlatform()
63
+elif is_cpu:
64
+ from .cpu import CpuPlatform
65
+ current_platform = CpuPlatform()
66
else:
67
current_platform = UnspecifiedPlatform()
68
vllm/platforms/cpu.py
new
+15
@@ -0,0 +1,15 @@
1
+import torch
2
+
3
+from .interface import Platform, PlatformEnum
4
+
5
+
6
+class CpuPlatform(Platform):
7
+ _enum = PlatformEnum.CPU
8
+
9
+ @staticmethod
10
+ def get_device_name(device_id: int = 0) -> str:
11
+ return "cpu"
12
+
13
+ @staticmethod
14
+ def inference_mode():
15
+ return torch.no_grad()
vllm/platforms/interface.py
+7
-3
@@ -1,5 +1,5 @@
1
import enum
2
-from typing import Tuple
2
+from typing import Optional, Tuple
3
4
import torch
5
@@ -8,6 +8,7 @@ class PlatformEnum(enum.Enum):
8
CUDA = enum.auto()
9
ROCM = enum.auto()
10
TPU = enum.auto()
11
+ CPU = enum.auto()
12
UNSPECIFIED = enum.auto()
13
14
@@ -23,9 +24,12 @@ class Platform:
24
def is_tpu(self) -> bool:
25
return self._enum == PlatformEnum.TPU
26
27
+ def is_cpu(self) -> bool:
28
+ return self._enum == PlatformEnum.CPU
29
+
30
@staticmethod
27
- def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
28
- raise NotImplementedError
31
+ def get_device_capability(device_id: int = 0) -> Optional[Tuple[int, int]]:
32
+ return None
33
34
@staticmethod
35
def get_device_name(device_id: int = 0) -> str:
vllm/platforms/tpu.py
-6
@@ -1,5 +1,3 @@
1
-from typing import Tuple
2
-
1
import torch
2
3
from .interface import Platform, PlatformEnum
@@ -8,10 +6,6 @@ from .interface import Platform, PlatformEnum
6
class TpuPlatform(Platform):
7
_enum = PlatformEnum.TPU
8
11
- @staticmethod
12
- def get_device_capability(device_id: int = 0) -> Tuple[int, int]:
13
- raise RuntimeError("TPU does not have device capability.")
14
-
9
@staticmethod
10
def inference_mode():
11
return torch.no_grad()
vllm/worker/cpu_worker.py
+2
-1
@@ -207,7 +207,8 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
207
208
def init_device(self) -> None:
209
if self.local_omp_cpuid != "all":
210
- torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)
210
+ ret = torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)
211
+ logger.info(ret)
212
213
self.init_distributed_environment()
214
# Set random seed.