A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Hardware][Intel] Add CPU inference backend (#3634)
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com> Co-authored-by: Yuan Zhou <yuan.zhou@intel.com>
bigPYJ1151 committed
Apr 2, 2024 at 13:07 UTC
0e3f06fe9ccfb59cb89c353b6ef81110fa0854c9
24 files changed
+2747
-5
.buildkite/run-cpu-test.sh
new
+14
@@ -0,0 +1,14 @@
1
+# This script build the CPU docker image and run the offline inference inside the container.
2
+# It serves a sanity check for compilation and basic model usage.
3
+set -ex
4
+
5
+# Try building the docker image
6
+docker build -t cpu-test -f Dockerfile.cpu .
7
+
8
+# Setup cleanup
9
+remove_docker_container() { docker rm -f cpu-test || true; }
10
+trap remove_docker_container EXIT
11
+remove_docker_container
12
+
13
+# Run the image and launch offline inference
14
+docker run --network host --env VLLM_CPU_KVCACHE_SPACE=1 --name cpu-check cpu-test python3 examples/offline_inference.py
.buildkite/test-template.j2
+3
@@ -8,6 +8,9 @@ steps:
8
queue: amd
9
command: bash .buildkite/run-amd-test.sh
10
11
+ - label: "CPU Test"
12
+ command: bash .buildkite/run-cpu-test.sh
13
+
14
- label: ":docker: build image"
15
commands:
16
- "docker build --build-arg max_jobs=16 --tag {{ docker_image }} --target test --progress plain ."
CMakeLists.txt
+16
@@ -2,7 +2,10 @@ cmake_minimum_required(VERSION 3.21)
2
3
project(vllm_extensions LANGUAGES CXX)
4
5
+option(VLLM_TARGET_DEVICE "Target device backend for vLLM" "cuda")
6
+
7
message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
8
+message(STATUS "Target device: ${VLLM_TARGET_DEVICE}")
9
10
include(${CMAKE_CURRENT_LIST_DIR}/cmake/utils.cmake)
11
@@ -76,6 +79,19 @@ find_package(Torch REQUIRED)
79
find_library(torch_python_LIBRARY torch_python PATHS
80
"${TORCH_INSTALL_PREFIX}/lib")
81
82
+#
83
+# Forward the non-CUDA device extensions to external CMake scripts.
84
+#
85
+if (NOT VLLM_TARGET_DEVICE STREQUAL "cuda" AND
86
+ NOT VLLM_TARGET_DEVICE STREQUAL "rocm")
87
+ if (VLLM_TARGET_DEVICE STREQUAL "cpu")
88
+ include(${CMAKE_CURRENT_LIST_DIR}/cmake/cpu_extension.cmake)
89
+ else()
90
+ message(FATAL_ERROR "Unsupported vLLM target device: ${VLLM_TARGET_DEVICE}")
91
+ endif()
92
+ return()
93
+endif()
94
+
95
#
96
# Set up GPU language and check the torch version and warn if it isn't
97
# what is expected.
Dockerfile.cpu
new
+20
@@ -0,0 +1,20 @@
1
+# This vLLM Dockerfile is used to construct image that can build and run vLLM on x86 CPU platform.
2
+
3
+FROM ubuntu:22.04
4
+
5
+RUN apt-get update -y \
6
+ && apt-get install -y git wget vim numactl gcc-12 g++-12 python3 python3-pip \
7
+ && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
8
+
9
+RUN pip install --upgrade pip \
10
+ && pip install wheel packaging ninja setuptools>=49.4.0 numpy
11
+
12
+COPY ./ /workspace/vllm
13
+
14
+WORKDIR /workspace/vllm
15
+
16
+RUN pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu
17
+
18
+RUN VLLM_TARGET_DEVICE=cpu python3 setup.py install
19
+
20
+CMD ["/bin/bash"]
cmake/cpu_extension.cmake
new
+90
@@ -0,0 +1,90 @@
1
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
2
+
3
+#
4
+# Define environment variables for special configurations
5
+#
6
+if(DEFINED ENV{VLLM_CPU_AVX512BF16})
7
+ set(ENABLE_AVX512BF16 ON)
8
+endif()
9
+
10
+include_directories("${CMAKE_SOURCE_DIR}/csrc")
11
+
12
+#
13
+# Check the compile flags
14
+#
15
+list(APPEND CXX_COMPILE_FLAGS
16
+ "-fopenmp"
17
+ "-DVLLM_CPU_EXTENSION")
18
+
19
+execute_process(COMMAND cat /proc/cpuinfo
20
+ RESULT_VARIABLE CPUINFO_RET
21
+ OUTPUT_VARIABLE CPUINFO)
22
+
23
+if (NOT CPUINFO_RET EQUAL 0)
24
+ message(FATAL_ERROR "Failed to check CPU features via /proc/cpuinfo")
25
+endif()
26
+
27
+function (find_isa CPUINFO TARGET OUT)
28
+ string(FIND ${CPUINFO} ${TARGET} ISA_FOUND)
29
+ if(NOT ISA_FOUND EQUAL -1)
30
+ set(${OUT} ON PARENT_SCOPE)
31
+ else()
32
+ set(${OUT} OFF PARENT_SCOPE)
33
+ endif()
34
+endfunction()
35
+
36
+find_isa(${CPUINFO} "avx512f" AVX512_FOUND)
37
+
38
+if (AVX512_FOUND)
39
+ list(APPEND CXX_COMPILE_FLAGS
40
+ "-mavx512f"
41
+ "-mavx512vl"
42
+ "-mavx512bw"
43
+ "-mavx512dq")
44
+
45
+ find_isa(${CPUINFO} "avx512_bf16" AVX512BF16_FOUND)
46
+ if (AVX512BF16_FOUND OR ENABLE_AVX512BF16)
47
+ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND
48
+ CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.3)
49
+ list(APPEND CXX_COMPILE_FLAGS "-mavx512bf16")
50
+ else()
51
+ message(WARNING "Disable AVX512-BF16 ISA support, requires gcc/g++ >= 12.3")
52
+ endif()
53
+ else()
54
+ message(WARNING "Disable AVX512-BF16 ISA support, no avx512_bf16 found in local CPU flags." " If cross-compilation is required, please set env VLLM_CPU_AVX512BF16=1.")
55
+ endif()
56
+else()
57
+ message(FATAL_ERROR "vLLM CPU backend requires AVX512 ISA support.")
58
+endif()
59
+
60
+message(STATUS "CPU extension compile flags: ${CXX_COMPILE_FLAGS}")
61
+
62
+
63
+#
64
+# Define extension targets
65
+#
66
+
67
+#
68
+# _C extension
69
+#
70
+set(VLLM_EXT_SRC
71
+ "csrc/cpu/activation.cpp"
72
+ "csrc/cpu/attention.cpp"
73
+ "csrc/cpu/cache.cpp"
74
+ "csrc/cpu/layernorm.cpp"
75
+ "csrc/cpu/pos_encoding.cpp"
76
+ "csrc/cpu/pybind.cpp")
77
+
78
+define_gpu_extension_target(
79
+ _C
80
+ DESTINATION vllm
81
+ LANGUAGE CXX
82
+ SOURCES ${VLLM_EXT_SRC}
83
+ COMPILE_FLAGS ${CXX_COMPILE_FLAGS}
84
+ WITH_SOABI
85
+)
86
+
87
+add_custom_target(default)
88
+message(STATUS "Enabling C extension.")
89
+add_dependencies(default _C)
90
+
csrc/cpu/activation.cpp
new
+148
@@ -0,0 +1,148 @@
1
+#include "cpu_types.hpp"
2
+
3
+namespace {
4
+template <typename scalar_t, vec_op::FP32Vec8 (*func)(const vec_op::FP32Vec8 &),
5
+ bool is_gated>
6
+void activation_kernel(int num_tokens, int d, scalar_t *__restrict__ input,
7
+ scalar_t *__restrict__ output) {
8
+ using scalar_vec_t = vec_op::vec_t<scalar_t>;
9
+ constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
10
+
11
+ TORCH_CHECK(d % VEC_ELEM_NUM == 0);
12
+
13
+#pragma omp parallel for
14
+ for (int i = 0; i < num_tokens; ++i) {
15
+ for (int j = 0; j < d; j += VEC_ELEM_NUM) {
16
+ int start = i * d;
17
+ if constexpr (is_gated) {
18
+ start *= 2;
19
+ }
20
+
21
+ const scalar_vec_t x(input + start + j);
22
+ const vec_op::FP32Vec8 f32_x(x);
23
+ vec_op::FP32Vec8 f32_ans = func(f32_x);
24
+
25
+ if constexpr (is_gated) {
26
+ const scalar_vec_t y(input + start + d + j);
27
+ const vec_op::FP32Vec8 f32_y(y);
28
+ f32_ans = f32_y * f32_ans;
29
+ }
30
+
31
+ const scalar_vec_t result(f32_ans);
32
+ result.save(output + i * d + j);
33
+ }
34
+ }
35
+}
36
+
37
+FORCE_INLINE vec_op::FP32Vec8 silu_act(const vec_op::FP32Vec8 &x) {
38
+ const vec_op::FP32Vec8 zeros(0.0);
39
+ const vec_op::FP32Vec8 ones(1.0);
40
+ return x / (ones + (zeros - x).exp());
41
+}
42
+
43
+FORCE_INLINE vec_op::FP32Vec8 gelu_new_act(const vec_op::FP32Vec8 &x) {
44
+ const vec_op::FP32Vec8 ones(1.0);
45
+ const vec_op::FP32Vec8 w1(0.79788456f);
46
+ const vec_op::FP32Vec8 w2(0.044715f);
47
+ const vec_op::FP32Vec8 w3(0.5);
48
+ const vec_op::FP32Vec8 x3 = x * x * x;
49
+ const vec_op::FP32Vec8 t = (w1 * (x + w2 * x3)).tanh();
50
+ return w3 * x * (ones + t);
51
+}
52
+
53
+FORCE_INLINE vec_op::FP32Vec8 gelu_fast_act(const vec_op::FP32Vec8 &x) {
54
+ const vec_op::FP32Vec8 ones(1.0);
55
+ const vec_op::FP32Vec8 w1(0.79788456f);
56
+ const vec_op::FP32Vec8 w2(0.044715f);
57
+ const vec_op::FP32Vec8 w3(0.5);
58
+ const vec_op::FP32Vec8 t = (x * w1 * (ones + x * w2 * x)).tanh();
59
+ return w3 * x * (ones + t);
60
+}
61
+
62
+FORCE_INLINE vec_op::FP32Vec8 gelu_act(const vec_op::FP32Vec8 &x) {
63
+ const vec_op::FP32Vec8 ones(1.0);
64
+ const vec_op::FP32Vec8 w1(M_SQRT1_2);
65
+ const vec_op::FP32Vec8 w2(0.5);
66
+ return x * w2 * (ones + (x * w1).er());
67
+}
68
+
69
+FORCE_INLINE vec_op::FP32Vec8 gelu_tanh_act(const vec_op::FP32Vec8 &x) {
70
+ const vec_op::FP32Vec8 ones(1.0);
71
+ const vec_op::FP32Vec8 w1(M_SQRT2 * M_2_SQRTPI * 0.5);
72
+ const vec_op::FP32Vec8 w2(0.5);
73
+ const vec_op::FP32Vec8 w3(0.044715);
74
+ const vec_op::FP32Vec8 x_3 = x * x * x;
75
+ const vec_op::FP32Vec8 inner = w1 * (x + x_3 * w3);
76
+ return x * w2 * (ones + inner.tanh());
77
+}
78
+}; // namespace
79
+
80
+void silu_and_mul(torch::Tensor &out, torch::Tensor &input) {
81
+ int num_tokens = input.numel() / input.size(-1);
82
+ int d = input.size(-1) / 2;
83
+
84
+ VLLM_DISPATCH_FLOATING_TYPES(
85
+ input.scalar_type(), "silu_and_mul_impl", [&] {
86
+ CPU_KERNEL_GUARD_IN(silu_and_mul_impl)
87
+ activation_kernel<scalar_t, silu_act, true>(num_tokens, d,
88
+ input.data_ptr<scalar_t>(),
89
+ out.data_ptr<scalar_t>());
90
+ CPU_KERNEL_GUARD_OUT(silu_and_mul_impl)
91
+ });
92
+}
93
+
94
+void gelu_and_mul(torch::Tensor &out, // [..., d]
95
+ torch::Tensor &input) // [..., 2 * d]
96
+{
97
+ int num_tokens = input.numel() / input.size(-1);
98
+ int d = input.size(-1) / 2;
99
+
100
+ VLLM_DISPATCH_FLOATING_TYPES(
101
+ input.scalar_type(), "gelu_and_mul_impl", [&] {
102
+ CPU_KERNEL_GUARD_IN(gelu_and_mul_impl)
103
+ activation_kernel<scalar_t, gelu_act, true>(num_tokens, d,
104
+ input.data_ptr<scalar_t>(),
105
+ out.data_ptr<scalar_t>());
106
+ CPU_KERNEL_GUARD_OUT(gelu_and_mul_impl)
107
+ });
108
+}
109
+
110
+void gelu_tanh_and_mul(torch::Tensor &out, // [..., d]
111
+ torch::Tensor &input) // [..., 2 * d]
112
+{
113
+ int num_tokens = input.numel() / input.size(-1);
114
+ int d = input.size(-1) / 2;
115
+
116
+ VLLM_DISPATCH_FLOATING_TYPES(
117
+ input.scalar_type(), "gelu_tanh_and_mul_impl", [&] {
118
+ CPU_KERNEL_GUARD_IN(gelu_tanh_and_mul_impl)
119
+ activation_kernel<scalar_t, gelu_tanh_act, true>(
120
+ num_tokens, d, input.data_ptr<scalar_t>(),
121
+ out.data_ptr<scalar_t>());
122
+ CPU_KERNEL_GUARD_OUT(gelu_tanh_and_mul_impl)
123
+ });
124
+}
125
+
126
+void gelu_new(torch::Tensor &out, torch::Tensor &input) {
127
+ int num_tokens = input.numel() / input.size(-1);
128
+ int d = input.size(-1);
129
+
130
+ VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_new_impl", [&] {
131
+ CPU_KERNEL_GUARD_IN(gelu_new_impl)
132
+ activation_kernel<scalar_t, gelu_new_act, false>(
133
+ num_tokens, d, input.data_ptr<scalar_t>(), out.data_ptr<scalar_t>());
134
+ CPU_KERNEL_GUARD_OUT(gelu_new_impl)
135
+ });
136
+}
137
+
138
+void gelu_fast(torch::Tensor &out, torch::Tensor &input) {
139
+ int num_tokens = input.numel() / input.size(-1);
140
+ int d = input.size(-1);
141
+
142
+ VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_fast_impl", [&] {
143
+ CPU_KERNEL_GUARD_IN(gelu_fast_impl)
144
+ activation_kernel<scalar_t, gelu_fast_act, false>(
145
+ num_tokens, d, input.data_ptr<scalar_t>(), out.data_ptr<scalar_t>());
146
+ CPU_KERNEL_GUARD_OUT(gelu_fast_impl)
147
+ });
148
+}
csrc/cpu/attention.cpp
new
+744
@@ -0,0 +1,744 @@
1
+#include "cpu_types.hpp"
2
+
3
+namespace {
4
+
5
+template <typename scalar_t> struct KernelVecType {
6
+ using q_load_vec_type = void;
7
+ using q_vec_type = void;
8
+ using k_load_vec_type = void;
9
+ using k_vec_type = void;
10
+ using qk_acc_vec_type = void;
11
+ using v_load_vec_type = void;
12
+};
13
+
14
+template <> struct KernelVecType<float> {
15
+ using q_load_vec_type = vec_op::FP32Vec4;
16
+ using q_vec_type = vec_op::FP32Vec16;
17
+ using k_load_vec_type = vec_op::FP32Vec16;
18
+ using k_vec_type = vec_op::FP32Vec16;
19
+ using qk_acc_vec_type = vec_op::FP32Vec16;
20
+ using v_load_vec_type = vec_op::FP32Vec16;
21
+};
22
+
23
+#ifdef __AVX512BF16__
24
+template <> struct KernelVecType<c10::BFloat16> {
25
+ using q_load_vec_type = vec_op::BF16Vec8;
26
+ using q_vec_type = vec_op::BF16Vec32;
27
+ using k_load_vec_type = vec_op::BF16Vec32;
28
+ using k_vec_type = vec_op::BF16Vec32;
29
+ using qk_acc_vec_type = vec_op::FP32Vec16;
30
+ using v_load_vec_type = vec_op::BF16Vec16;
31
+};
32
+#else
33
+template <> struct KernelVecType<c10::BFloat16> {
34
+ using q_load_vec_type = vec_op::BF16Vec8;
35
+ using q_vec_type = vec_op::FP32Vec16;
36
+ using k_load_vec_type = vec_op::BF16Vec16;
37
+ using k_vec_type = vec_op::FP32Vec16;
38
+ using qk_acc_vec_type = vec_op::FP32Vec16;
39
+ using v_load_vec_type = vec_op::BF16Vec16;
40
+};
41
+#endif
42
+
43
+template <typename T>
44
+FORCE_INLINE std::pair<T, T> reduceSoftmax(T *data, const int size,
45
+ const int capacity) {
46
+ T max = data[0];
47
+ for (int i = 1; i < size; ++i) {
48
+ max = max >= data[i] ? max : data[i];
49
+ }
50
+
51
+ T sum = 0;
52
+ for (int i = 0; i < size; ++i) {
53
+ data[i] = std::exp(data[i] - max);
54
+ sum += data[i];
55
+ }
56
+
57
+ int i = 0;
58
+ for (; i < size; ++i) {
59
+ data[i] /= sum;
60
+ }
61
+
62
+ for (; i < capacity; ++i) {
63
+ data[i] = 0;
64
+ }
65
+
66
+ return {max, sum};
67
+}
68
+
69
+template <typename T>
70
+FORCE_INLINE std::pair<T, T>
71
+reduceSoftmaxAlibi(T *data, const int size, const int capacity,
72
+ const float alibi_slope, const int start_index,
73
+ const int context_len) {
74
+ data[0] += alibi_slope * (start_index - context_len + 1);
75
+ T max = data[0];
76
+ for (int i = 1; i < size; ++i) {
77
+ T qk = data[i] + alibi_slope * (start_index + i - context_len + 1);
78
+ data[i] = qk;
79
+ max = max >= qk ? max : qk;
80
+ }
81
+
82
+ T sum = 0;
83
+ for (int i = 0; i < size; ++i) {
84
+ data[i] = std::exp(data[i] - max);
85
+ sum += data[i];
86
+ }
87
+
88
+ int i = 0;
89
+ for (; i < size; ++i) {
90
+ data[i] /= sum;
91
+ }
92
+
93
+ for (; i < capacity; ++i) {
94
+ data[i] = 0;
95
+ }
96
+
97
+ return {max, sum};
98
+}
99
+
100
+template <typename T>
101
+FORCE_INLINE void reducePartitonSoftmax(const T *max_data, T *sum_data,
102
+ const int size) {
103
+ T max = max_data[0];
104
+ for (int i = 1; i < size; ++i) {
105
+ max = max >= max_data[i] ? max : max_data[i];
106
+ }
107
+
108
+ T rescaled_sum = 0;
109
+ for (int i = 0; i < size; ++i) {
110
+ T rescale_factor = std::exp(max_data[i] - max);
111
+ rescaled_sum += rescale_factor * sum_data[i];
112
+ sum_data[i] *= rescale_factor;
113
+ }
114
+ for (int i = 0; i < size; ++i) {
115
+ sum_data[i] /= rescaled_sum + 1e-8;
116
+ }
117
+}
118
+
119
+template <typename scalar_t, int HEAD_SIZE, int BLOCK_SIZE, int x>
120
+struct reduceQKBlockKernel {
121
+ using q_load_vec_type = typename KernelVecType<scalar_t>::q_load_vec_type;
122
+ using q_vec_type = typename KernelVecType<scalar_t>::q_vec_type;
123
+ using k_load_vec_type = typename KernelVecType<scalar_t>::k_load_vec_type;
124
+ using k_vec_type = typename KernelVecType<scalar_t>::k_vec_type;
125
+ using qk_acc_vec_type = typename KernelVecType<scalar_t>::qk_acc_vec_type;
126
+
127
+ constexpr static int TOKEN_PER_GROUP = k_load_vec_type::get_elem_num() / x;
128
+ constexpr static int MAX_GROUP_NUM = 16 / TOKEN_PER_GROUP;
129
+ constexpr static int UNROLL_GROUP_NUM = MAX_GROUP_NUM / 4;
130
+
131
+ static_assert(MAX_GROUP_NUM == 8 || MAX_GROUP_NUM == 4);
132
+ static_assert(k_load_vec_type::get_elem_num() % x == 0);
133
+ static_assert(q_load_vec_type::get_elem_num() * sizeof(scalar_t) == 16);
134
+
135
+ FORCE_INLINE static void call(const scalar_t *__restrict__ q,
136
+ const scalar_t *__restrict__ k_block,
137
+ float *__restrict__ logits, float scale,
138
+ const int token_num) {
139
+ const int group_num = (token_num + TOKEN_PER_GROUP - 1) / TOKEN_PER_GROUP;
140
+
141
+ qk_acc_vec_type group_accums[MAX_GROUP_NUM];
142
+ if (token_num == BLOCK_SIZE) {
143
+ for (int q_offset = 0; q_offset < HEAD_SIZE;
144
+ q_offset += x, k_block += x * BLOCK_SIZE) {
145
+ q_load_vec_type q_load_group_vec(q + q_offset);
146
+ q_vec_type q_group_vec(q_load_group_vec);
147
+
148
+ vec_op::unroll_loop<int, MAX_GROUP_NUM>(
149
+ [k_block, &q_group_vec, &group_accums](int token_group_idx) {
150
+ k_load_vec_type k_load_group_vec(k_block + token_group_idx * x *
151
+ TOKEN_PER_GROUP);
152
+ k_vec_type k_group_vec(k_load_group_vec);
153
+ vec_op::fma(group_accums[token_group_idx], q_group_vec,
154
+ k_group_vec);
155
+ vec_op::prefetch(k_block + x * BLOCK_SIZE +
156
+ token_group_idx * x * TOKEN_PER_GROUP);
157
+ });
158
+ }
159
+ } else {
160
+ for (int q_offset = 0; q_offset < HEAD_SIZE;
161
+ q_offset += x, k_block += x * BLOCK_SIZE) {
162
+ q_load_vec_type q_load_group_vec(q + q_offset);
163
+ q_vec_type q_group_vec(q_load_group_vec);
164
+ for (int token_group_start = 0; token_group_start < group_num;
165
+ token_group_start += UNROLL_GROUP_NUM) {
166
+ vec_op::unroll_loop<int, UNROLL_GROUP_NUM>(
167
+ [token_group_start, k_block, &q_group_vec,
168
+ &group_accums](int token_group_idx) {
169
+ token_group_idx += token_group_start;
170
+ k_load_vec_type k_load_group_vec(k_block + token_group_idx * x *
171
+ TOKEN_PER_GROUP);
172
+ k_vec_type k_group_vec(k_load_group_vec);
173
+ vec_op::fma(group_accums[token_group_idx], q_group_vec,
174
+ k_group_vec);
175
+ vec_op::prefetch(k_block + x * BLOCK_SIZE +
176
+ token_group_idx * x * TOKEN_PER_GROUP);
177
+ });
178
+ }
179
+ }
180
+ }
181
+
182
+ for (int token_group_idx = 0; token_group_idx < group_num;
183
+ ++token_group_idx) {
184
+ vec_op::unroll_loop<int, TOKEN_PER_GROUP>(
185
+ [&group_accums, logits, scale, token_group_idx](int token_idx) {
186
+ float dot_v =
187
+ group_accums[token_group_idx]
188
+ .template reduce_sub_sum<qk_acc_vec_type::get_elem_num() /
189
+ TOKEN_PER_GROUP>(token_idx);
190
+ logits[token_group_idx * TOKEN_PER_GROUP + token_idx] =
191
+ dot_v * scale;
192
+ });
193
+ }
194
+ }
195
+};
196
+
197
+template <typename scalar_t, int HEAD_SIZE, int BLOCK_SIZE,
198
+ int HEAD_PARTITION_SIZE, typename acc_t>
199
+FORCE_INLINE void reduceValueBlock(const float *prob, const scalar_t *v_block,
200
+ acc_t &&acc) {
201
+ using v_load_vec_type = typename KernelVecType<scalar_t>::v_load_vec_type;
202
+ constexpr int ELEM_NUM = v_load_vec_type::get_elem_num();
203
+ static_assert(BLOCK_SIZE == ELEM_NUM);
204
+ vec_op::FP32Vec16 prob_vec(prob);
205
+
206
+ vec_op::unroll_loop<int, HEAD_PARTITION_SIZE>([&](int head_elem_idx) {
207
+ v_load_vec_type v_vec(v_block + BLOCK_SIZE * head_elem_idx);
208
+ vec_op::FP32Vec16 fp32_v_vec(v_vec);
209
+ acc[head_elem_idx] = acc[head_elem_idx] + prob_vec * fp32_v_vec;
210
+ });
211
+}
212
+}; // namespace
213
+
214
+// Paged attention v1
215
+namespace {
216
+template <typename scalar_t, int HEAD_SIZE, int BLOCK_SIZE>
217
+struct paged_attention_v1_impl {
218
+ static void
219
+ call(scalar_t *__restrict__ out, // [num_seqs, num_heads, head_size]
220
+ const scalar_t *__restrict__ q, // [num_seqs, num_heads, head_size]
221
+ const scalar_t *__restrict__ k_cache, // [num_blocks, num_kv_heads,
222
+ // head_size/x, block_size, x]
223
+ const scalar_t *__restrict__ v_cache, // [num_blocks, num_kv_heads,
224
+ // head_size, block_size]
225
+ const int num_kv_heads, const float scale,
226
+ const int
227
+ *__restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
228
+ const int *__restrict__ context_lens, // [num_seqs]
229
+ const int max_num_blocks_per_seq,
230
+ const float *__restrict__ alibi_slopes, // [num_heads]
231
+ const int q_stride, const int kv_block_stride, const int kv_head_stride,
232
+ const int num_seqs, const int num_heads) {
233
+ constexpr int x = 16 / sizeof(scalar_t);
234
+ const int num_queries_per_kv = num_heads / num_kv_heads;
235
+
236
+ static_assert(BLOCK_SIZE == 16);
237
+
238
+ int max_context_len = max_num_blocks_per_seq * BLOCK_SIZE;
239
+ int max_context_len_padded = (max_context_len + 15) & 0xFFFFFFF0;
240
+ TORCH_CHECK((max_context_len_padded * sizeof(float)) % 64 == 0);
241
+
242
+ const int parallel_work_item_num = omp_get_max_threads();
243
+
244
+ size_t logits_bytes =
245
+ parallel_work_item_num * max_context_len_padded * sizeof(float);
246
+ float *logits = (float *)std::aligned_alloc(
247
+ 64, logits_bytes); // Cacheline alignment for each context token.
248
+ // [parallel_work_item_num, max_context_len_padded]
249
+
250
+#pragma omp parallel for collapse(2) schedule(dynamic, 1)
251
+ for (int seq_idx = 0; seq_idx < num_seqs; ++seq_idx) {
252
+ for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
253
+ int context_len = context_lens[seq_idx];
254
+ const int *seq_block_table =
255
+ block_tables + max_num_blocks_per_seq * seq_idx;
256
+ const int block_num = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
257
+ const int64_t kv_head_idx = head_idx / num_queries_per_kv;
258
+ const scalar_t *__restrict__ q_vec_ptr =
259
+ q + seq_idx * q_stride + head_idx * HEAD_SIZE;
260
+ const int last_block_token_num =
261
+ context_len - (block_num - 1) * BLOCK_SIZE;
262
+ float *__restrict__ thread_block_logits =
263
+ logits + omp_get_thread_num() * max_context_len_padded;
264
+
265
+ // Compute logits
266
+ for (int block_idx = 0; block_idx < block_num; ++block_idx) {
267
+ const int64_t physical_block_idx = seq_block_table[block_idx];
268
+ const scalar_t *__restrict__ k_block_cache_ptr =
269
+ k_cache + physical_block_idx * kv_block_stride +
270
+ kv_head_idx * kv_head_stride;
271
+ float *__restrict__ head_block_logits =
272
+ thread_block_logits + block_idx * BLOCK_SIZE;
273
+
274
+ reduceQKBlockKernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, x>::call(
275
+ q_vec_ptr, k_block_cache_ptr, head_block_logits, scale,
276
+ block_idx == block_num - 1 ? last_block_token_num : BLOCK_SIZE);
277
+ }
278
+
279
+ // Compute softmax
280
+ if (alibi_slopes) {
281
+ reduceSoftmaxAlibi(thread_block_logits, context_len,
282
+ block_num * BLOCK_SIZE, alibi_slopes[head_idx], 0,
283
+ context_len);
284
+ } else {
285
+ reduceSoftmax(thread_block_logits, context_len,
286
+ block_num * BLOCK_SIZE);
287
+ }
288
+
289
+ // Compute value
290
+ constexpr int head_elem_num_per_partition = 16;
291
+ constexpr int head_partition_num =
292
+ HEAD_SIZE / head_elem_num_per_partition;
293
+ for (int head_part_idx = 0; head_part_idx < head_partition_num;
294
+ ++head_part_idx) {
295
+ vec_op::FP32Vec16 accums[head_elem_num_per_partition];
296
+ scalar_t *__restrict__ out_ptr =
297
+ out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE +
298
+ head_part_idx * head_elem_num_per_partition;
299
+ for (int block_idx = 0; block_idx < block_num; ++block_idx) {
300
+ const int64_t physical_block_idx = seq_block_table[block_idx];
301
+ const float *__restrict__ prob_vec_ptr =
302
+ thread_block_logits + block_idx * BLOCK_SIZE;
303
+ const scalar_t *__restrict__ v_block_cache_ptr =
304
+ v_cache + physical_block_idx * kv_block_stride +
305
+ kv_head_idx * kv_head_stride +
306
+ BLOCK_SIZE * head_part_idx * head_elem_num_per_partition;
307
+ reduceValueBlock<scalar_t, HEAD_SIZE, BLOCK_SIZE,
308
+ head_elem_num_per_partition>(
309
+ prob_vec_ptr, v_block_cache_ptr, accums);
310
+
311
+ if (block_idx != block_num - 1) {
312
+ const int64_t next_physical_block_idx =
313
+ seq_block_table[block_idx + 1];
314
+ const scalar_t *__restrict__ next_v_block_cache_ptr =
315
+ v_cache + next_physical_block_idx * kv_block_stride +
316
+ kv_head_idx * kv_head_stride +
317
+ BLOCK_SIZE * head_part_idx * head_elem_num_per_partition;
318
+ vec_op::unroll_loop<int, head_elem_num_per_partition>(
319
+ [&](int head_elem_idx) {
320
+ if (head_elem_idx % 2 == 0) {
321
+ vec_op::prefetch(next_v_block_cache_ptr +
322
+ BLOCK_SIZE * head_elem_idx);
323
+ }
324
+ });
325
+ }
326
+ }
327
+
328
+ vec_op::unroll_loop<int, head_elem_num_per_partition>(
329
+ [&](int head_elem_idx) {
330
+ float value = accums[head_elem_idx].reduce_sum();
331
+ vec_op::storeFP32(value, out_ptr + head_elem_idx);
332
+ });
333
+ }
334
+ }
335
+ }
336
+ std::free(logits);
337
+ }
338
+};
339
+
340
+#define LAUNCH_V1_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE) \
341
+ paged_attention_v1_impl<T, HEAD_SIZE, BLOCK_SIZE>::call( \
342
+ out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, scale, \
343
+ block_tables_ptr, context_lens_ptr, max_num_blocks_per_seq, \
344
+ alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, num_seqs, \
345
+ num_heads);
346
+
347
+template <typename T, int BLOCK_SIZE>
348
+void paged_attention_v1_impl_launcher(
349
+ torch::Tensor &out, torch::Tensor &query, torch::Tensor &key_cache,
350
+ torch::Tensor &value_cache, int num_kv_heads, float scale,
351
+ torch::Tensor &block_tables, torch::Tensor &context_lens,
352
+ int max_context_len, const c10::optional<torch::Tensor> &alibi_slopes) {
353
+ int num_seqs = query.size(0);
354
+ int num_heads = query.size(1);
355
+ int head_size = query.size(2);
356
+ int max_num_blocks_per_seq = block_tables.size(1);
357
+ int q_stride = query.stride(0);
358
+ int kv_block_stride = key_cache.stride(0);
359
+ int kv_head_stride = key_cache.stride(1);
360
+
361
+ // NOTE: alibi_slopes is optional.
362
+ const float *alibi_slopes_ptr =
363
+ alibi_slopes
364
+ ? reinterpret_cast<const float *>(alibi_slopes.value().data_ptr())
365
+ : nullptr;
366
+
367
+ T *out_ptr = reinterpret_cast<T *>(out.data_ptr());
368
+ T *query_ptr = reinterpret_cast<T *>(query.data_ptr());
369
+ T *key_cache_ptr = reinterpret_cast<T *>(key_cache.data_ptr());
370
+ T *value_cache_ptr = reinterpret_cast<T *>(value_cache.data_ptr());
371
+ int *block_tables_ptr = block_tables.data_ptr<int>();
372
+ int *context_lens_ptr = context_lens.data_ptr<int>();
373
+
374
+ switch (head_size) {
375
+ case 64:
376
+ LAUNCH_V1_ATTENTION_KERNEL(T, 64, BLOCK_SIZE);
377
+ break;
378
+ case 80:
379
+ LAUNCH_V1_ATTENTION_KERNEL(T, 80, BLOCK_SIZE);
380
+ break;
381
+ case 96:
382
+ LAUNCH_V1_ATTENTION_KERNEL(T, 96, BLOCK_SIZE);
383
+ break;
384
+ case 112:
385
+ LAUNCH_V1_ATTENTION_KERNEL(T, 112, BLOCK_SIZE);
386
+ break;
387
+ case 128:
388
+ LAUNCH_V1_ATTENTION_KERNEL(T, 128, BLOCK_SIZE);
389
+ break;
390
+ case 256:
391
+ LAUNCH_V1_ATTENTION_KERNEL(T, 256, BLOCK_SIZE);
392
+ break;
393
+ default:
394
+ TORCH_CHECK(false, "Unsupported head size: ", head_size);
395
+ break;
396
+ }
397
+}
398
+
399
+#define CALL_V1_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
400
+ paged_attention_v1_impl_launcher<T, BLOCK_SIZE>( \
401
+ out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
402
+ context_lens, max_context_len, alibi_slopes);
403
+
404
+#define CALL_V1_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
405
+ switch (block_size) { \
406
+ case 16: \
407
+ CALL_V1_KERNEL_LAUNCHER(T, 16); \
408
+ break; \
409
+ default: \
410
+ TORCH_CHECK(false, "Unsupported block size: ", block_size); \
411
+ break; \
412
+ }
413
+} // namespace
414
+
415
+void paged_attention_v1(torch::Tensor &out, torch::Tensor &query,
416
+ torch::Tensor &key_cache, torch::Tensor &value_cache,
417
+ int num_kv_heads, float scale,
418
+ torch::Tensor &block_tables,
419
+ torch::Tensor &context_lens, int block_size,
420
+ int max_context_len,
421
+ const c10::optional<torch::Tensor> &alibi_slopes,
422
+ const std::string &kv_cache_dtype) {
423
+ VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v1_impl",
424
+ [&] {
425
+ CPU_KERNEL_GUARD_IN(paged_attention_v1_impl)
426
+ CALL_V1_KERNEL_LAUNCHER_BLOCK_SIZE(scalar_t);
427
+ CPU_KERNEL_GUARD_OUT(paged_attention_v1_impl)
428
+ });
429
+}
430
+
431
+// Paged attention v2
432
+namespace {
433
+template <typename scalar_t, int HEAD_SIZE, int BLOCK_SIZE, int PARTITION_SIZE>
434
+struct paged_attention_v2_impl {
435
+ static void call(
436
+ scalar_t *__restrict__ out, // [num_seqs, num_heads, head_size]
437
+ float *__restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
438
+ float
439
+ *__restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
440
+ scalar_t *__restrict__ tmp_out, // [num_seqs, num_heads,
441
+ // max_num_partitions, head_size]
442
+ const scalar_t *__restrict__ q, // [num_seqs, num_heads, head_size]
443
+ const scalar_t *__restrict__ k_cache, // [num_blocks, num_kv_heads,
444
+ // head_size/x, block_size, x]
445
+ const scalar_t *__restrict__ v_cache, // [num_blocks, num_kv_heads,
446
+ // head_size, block_size]
447
+ const int num_kv_heads, const float scale,
448
+ const int
449
+ *__restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
450
+ const int *__restrict__ context_lens, // [num_seqs]
451
+ const int max_num_blocks_per_seq,
452
+ const float *__restrict__ alibi_slopes, // [num_heads]
453
+ const int q_stride, const int kv_block_stride, const int kv_head_stride,
454
+ const int num_seqs, const int num_heads, const int max_num_partitions) {
455
+ constexpr int x = 16 / sizeof(scalar_t);
456
+ const int num_queries_per_kv = num_heads / num_kv_heads;
457
+
458
+ static_assert(BLOCK_SIZE == 16);
459
+ static_assert(PARTITION_SIZE * sizeof(float) % 64 == 0);
460
+ static_assert(PARTITION_SIZE % BLOCK_SIZE == 0);
461
+
462
+#pragma omp parallel for collapse(3) schedule(static, 1)
463
+ for (int seq_idx = 0; seq_idx < num_seqs; ++seq_idx) {
464
+ for (int partition_idx = 0; partition_idx < max_num_partitions;
465
+ ++partition_idx) {
466
+ for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
467
+ const int context_len = context_lens[seq_idx];
468
+ const int start_token_idx = partition_idx * PARTITION_SIZE;
469
+
470
+ if (start_token_idx >= context_len)
471
+ continue;
472
+
473
+ const int partition_num =
474
+ (context_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
475
+ const bool no_reduce = (partition_num == 1);
476
+ const int context_token_num =
477
+ (std::min(context_len, start_token_idx + PARTITION_SIZE) -
478
+ start_token_idx);
479
+ const int block_num =
480
+ (context_token_num + BLOCK_SIZE - 1) / BLOCK_SIZE;
481
+ const int last_block_token_num =
482
+ context_token_num - (block_num - 1) * BLOCK_SIZE;
483
+ const int *seq_block_table = block_tables +
484
+ max_num_blocks_per_seq * seq_idx +
485
+ start_token_idx / BLOCK_SIZE;
486
+ const int64_t kv_head_idx = head_idx / num_queries_per_kv;
487
+ const scalar_t *__restrict__ q_vec_ptr =
488
+ q + seq_idx * q_stride + head_idx * HEAD_SIZE;
489
+
490
+ float logits[PARTITION_SIZE] __attribute__((aligned(64))) = {0};
491
+
492
+ // Compute logits
493
+ for (int block_idx = 0; block_idx < block_num; ++block_idx) {
494
+ const int64_t physical_block_idx = seq_block_table[block_idx];
495
+ const scalar_t *__restrict__ k_block_cache_ptr =
496
+ k_cache + physical_block_idx * kv_block_stride +
497
+ kv_head_idx * kv_head_stride;
498
+ float *__restrict__ head_block_logits =
499
+ logits + block_idx * BLOCK_SIZE;
500
+
501
+ reduceQKBlockKernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, x>::call(
502
+ q_vec_ptr, k_block_cache_ptr, head_block_logits, scale,
503
+ block_idx == block_num - 1 ? last_block_token_num : BLOCK_SIZE);
504
+ }
505
+
506
+ std::pair<float, float> max_and_sum;
507
+ if (alibi_slopes) {
508
+ max_and_sum = reduceSoftmaxAlibi(
509
+ logits, context_token_num, block_num * BLOCK_SIZE,
510
+ alibi_slopes[head_idx], start_token_idx, context_len);
511
+ } else {
512
+ max_and_sum = reduceSoftmax(logits, context_token_num,
513
+ block_num * BLOCK_SIZE);
514
+ }
515
+
516
+ auto &&[max_logit, exp_sum] = max_and_sum;
517
+
518
+ scalar_t *__restrict__ output_buffer = nullptr;
519
+ if (!no_reduce) {
520
+ auto idx = seq_idx * num_heads * max_num_partitions +
521
+ head_idx * max_num_partitions + partition_idx;
522
+ max_logits[idx] = max_logit;
523
+ exp_sums[idx] = exp_sum;
524
+ output_buffer =
525
+ tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE +
526
+ head_idx * max_num_partitions * HEAD_SIZE +
527
+ partition_idx * HEAD_SIZE;
528
+ } else {
529
+ output_buffer =
530
+ out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
531
+ }
532
+
533
+ // Compute value
534
+ constexpr int head_elem_num_per_partition = 16;
535
+ constexpr int head_partition_num =
536
+ HEAD_SIZE / head_elem_num_per_partition;
537
+ for (int head_part_idx = 0; head_part_idx < head_partition_num;
538
+ ++head_part_idx) {
539
+ vec_op::FP32Vec16 accums[head_elem_num_per_partition];
540
+ scalar_t *__restrict__ out_ptr =
541
+ output_buffer + head_part_idx * head_elem_num_per_partition;
542
+ for (int block_idx = 0; block_idx < block_num; ++block_idx) {
543
+ const int64_t physical_block_idx = seq_block_table[block_idx];
544
+ const float *__restrict__ prob_vec_ptr =
545
+ logits + block_idx * BLOCK_SIZE;
546
+ const scalar_t *__restrict__ v_block_cache_ptr =
547
+ v_cache + physical_block_idx * kv_block_stride +
548
+ kv_head_idx * kv_head_stride +
549
+ BLOCK_SIZE * head_part_idx * head_elem_num_per_partition;
550
+ reduceValueBlock<scalar_t, HEAD_SIZE, BLOCK_SIZE,
551
+ head_elem_num_per_partition>(
552
+ prob_vec_ptr, v_block_cache_ptr, accums);
553
+
554
+ if (block_idx != block_num - 1) {
555
+ const int64_t next_physical_block_idx =
556
+ seq_block_table[block_idx + 1];
557
+ const scalar_t *__restrict__ next_v_block_cache_ptr =
558
+ v_cache + next_physical_block_idx * kv_block_stride +
559
+ kv_head_idx * kv_head_stride +
560
+ BLOCK_SIZE * head_part_idx * head_elem_num_per_partition;
561
+ vec_op::unroll_loop<int, head_elem_num_per_partition>(
562
+ [&](int head_elem_idx) {
563
+ if (head_elem_idx % 2 == 0) {
564
+ vec_op::prefetch(next_v_block_cache_ptr +
565
+ BLOCK_SIZE * head_elem_idx);
566
+ }
567
+ });
568
+ }
569
+ }
570
+
571
+ vec_op::unroll_loop<int, head_elem_num_per_partition>(
572
+ [&](int head_elem_idx) {
573
+ float value = accums[head_elem_idx].reduce_sum();
574
+ vec_op::storeFP32(value, out_ptr + head_elem_idx);
575
+ });
576
+ }
577
+ }
578
+ }
579
+ }
580
+
581
+ // Rescale partition softmax and store the factors to exp_sums
582
+#pragma omp parallel for collapse(2) schedule(static, 1)
583
+ for (int seq_idx = 0; seq_idx < num_seqs; ++seq_idx) {
584
+ for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
585
+ const int context_len = context_lens[seq_idx];
586
+ const int partition_num =
587
+ (context_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
588
+
589
+ if (partition_num == 1)
590
+ continue;
591
+
592
+ reducePartitonSoftmax(
593
+ max_logits + seq_idx * num_heads * max_num_partitions +
594
+ head_idx * max_num_partitions,
595
+ exp_sums + seq_idx * num_heads * max_num_partitions +
596
+ head_idx * max_num_partitions,
597
+ partition_num);
598
+ }
599
+ }
600
+
601
+ // Reduce values
602
+ using v_load_vec_type = typename KernelVecType<scalar_t>::v_load_vec_type;
603
+ static_assert(v_load_vec_type::get_elem_num() == BLOCK_SIZE);
604
+ constexpr int head_elem_num_per_group =
605
+ 16; // Note: didn't align with the cacheline size, due to some HEAD_SIZE
606
+ // didn't align with 64 bytes
607
+ static_assert(HEAD_SIZE % head_elem_num_per_group == 0);
608
+ constexpr int head_group_num = HEAD_SIZE / head_elem_num_per_group;
609
+ const float *__restrict__ rescale_factors = exp_sums;
610
+#pragma omp parallel for collapse(3) schedule(static, 1)
611
+ for (int seq_idx = 0; seq_idx < num_seqs; ++seq_idx) {
612
+ for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
613
+ for (int group_idx = 0; group_idx < head_group_num; ++group_idx) {
614
+ const int context_len = context_lens[seq_idx];
615
+ const int partition_num =
616
+ (context_len + PARTITION_SIZE - 1) / PARTITION_SIZE;
617
+
618
+ if (partition_num == 1)
619
+ continue;
620
+
621
+ const float *__restrict__ seq_head_rescale_factors =
622
+ rescale_factors + seq_idx * num_heads * max_num_partitions +
623
+ head_idx * max_num_partitions;
624
+ const scalar_t *__restrict__ seq_head_tmp_out =
625
+ tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE +
626
+ head_idx * max_num_partitions * HEAD_SIZE +
627
+ group_idx * head_elem_num_per_group;
628
+ scalar_t *__restrict__ seq_head_output =
629
+ out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE +
630
+ group_idx * head_elem_num_per_group;
631
+
632
+ vec_op::FP32Vec16 acc;
633
+ for (int i = 0; i < partition_num; ++i) {
634
+ vec_op::FP32Vec16 rescale_factor(seq_head_rescale_factors[i]);
635
+ v_load_vec_type value(seq_head_tmp_out + i * HEAD_SIZE);
636
+ vec_op::FP32Vec16 fp32_value(value);
637
+ acc = acc + fp32_value * rescale_factor;
638
+ }
639
+ v_load_vec_type cast_acc(acc);
640
+ cast_acc.save(seq_head_output);
641
+ }
642
+ }
643
+ }
644
+ }
645
+};
646
+
647
+#define LAUNCH_V2_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE) \
648
+ paged_attention_v2_impl<T, HEAD_SIZE, BLOCK_SIZE, PARTITION_SIZE>::call( \
649
+ out_ptr, exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, \
650
+ key_cache_ptr, value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
651
+ context_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
652
+ kv_block_stride, kv_head_stride, num_seqs, num_heads, \
653
+ max_num_partitions);
654
+
655
+template <typename T, int BLOCK_SIZE, int PARTITION_SIZE = 512>
656
+void paged_attention_v2_impl_launcher(
657
+ torch::Tensor &out, torch::Tensor &exp_sums, torch::Tensor &max_logits,
658
+ torch::Tensor &tmp_out, torch::Tensor &query, torch::Tensor &key_cache,
659
+ torch::Tensor &value_cache, int num_kv_heads, float scale,
660
+ torch::Tensor &block_tables, torch::Tensor &context_lens, int block_size,
661
+ int max_context_len, const c10::optional<torch::Tensor> &alibi_slopes) {
662
+ int num_seqs = query.size(0);
663
+ int num_heads = query.size(1);
664
+ int head_size = query.size(2);
665
+ int max_num_blocks_per_seq = block_tables.size(1);
666
+ int q_stride = query.stride(0);
667
+ int kv_block_stride = key_cache.stride(0);
668
+ int kv_head_stride = key_cache.stride(1);
669
+ int max_num_partitions = exp_sums.size(-1);
670
+
671
+ // NOTE: alibi_slopes is optional.
672
+ const float *alibi_slopes_ptr =
673
+ alibi_slopes
674
+ ? reinterpret_cast<const float *>(alibi_slopes.value().data_ptr())
675
+ : nullptr;
676
+
677
+ T *out_ptr = reinterpret_cast<T *>(out.data_ptr());
678
+ float *exp_sums_ptr = reinterpret_cast<float *>(exp_sums.data_ptr());
679
+ float *max_logits_ptr = reinterpret_cast<float *>(max_logits.data_ptr());
680
+ T *tmp_out_ptr = reinterpret_cast<T *>(tmp_out.data_ptr());
681
+ T *query_ptr = reinterpret_cast<T *>(query.data_ptr());
682
+ T *key_cache_ptr = reinterpret_cast<T *>(key_cache.data_ptr());
683
+ T *value_cache_ptr = reinterpret_cast<T *>(value_cache.data_ptr());
684
+ int *block_tables_ptr = block_tables.data_ptr<int>();
685
+ int *context_lens_ptr = context_lens.data_ptr<int>();
686
+
687
+ switch (head_size) {
688
+ case 64:
689
+ LAUNCH_V2_ATTENTION_KERNEL(T, 64, BLOCK_SIZE);
690
+ break;
691
+ case 80:
692
+ LAUNCH_V2_ATTENTION_KERNEL(T, 80, BLOCK_SIZE);
693
+ break;
694
+ case 96:
695
+ LAUNCH_V2_ATTENTION_KERNEL(T, 96, BLOCK_SIZE);
696
+ break;
697
+ case 112:
698
+ LAUNCH_V2_ATTENTION_KERNEL(T, 112, BLOCK_SIZE);
699
+ break;
700
+ case 128:
701
+ LAUNCH_V2_ATTENTION_KERNEL(T, 128, BLOCK_SIZE);
702
+ break;
703
+ case 256:
704
+ LAUNCH_V2_ATTENTION_KERNEL(T, 256, BLOCK_SIZE);
705
+ break;
706
+ default:
707
+ TORCH_CHECK(false, "Unsupported head size: ", head_size);
708
+ break;
709
+ }
710
+}
711
+
712
+#define CALL_V2_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
713
+ paged_attention_v2_impl_launcher<T, BLOCK_SIZE>( \
714
+ out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
715
+ num_kv_heads, scale, block_tables, context_lens, block_size, \
716
+ max_context_len, alibi_slopes);
717
+
718
+#define CALL_V2_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
719
+ switch (block_size) { \
720
+ case 16: \
721
+ CALL_V2_KERNEL_LAUNCHER(T, 16); \
722
+ break; \
723
+ default: \
724
+ TORCH_CHECK(false, "Unsupported block size: ", block_size); \
725
+ break; \
726
+ }
727
+} // namespace
728
+
729
+void paged_attention_v2(torch::Tensor &out, torch::Tensor &exp_sums,
730
+ torch::Tensor &max_logits, torch::Tensor &tmp_out,
731
+ torch::Tensor &query, torch::Tensor &key_cache,
732
+ torch::Tensor &value_cache, int num_kv_heads,
733
+ float scale, torch::Tensor &block_tables,
734
+ torch::Tensor &context_lens, int block_size,
735
+ int max_context_len,
736
+ const c10::optional<torch::Tensor> &alibi_slopes,
737
+ const std::string &kv_cache_dtype) {
738
+ VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v2_impl",
739
+ [&] {
740
+ CPU_KERNEL_GUARD_IN(paged_attention_v2_impl)
741
+ CALL_V2_KERNEL_LAUNCHER_BLOCK_SIZE(scalar_t);
742
+ CPU_KERNEL_GUARD_OUT(paged_attention_v2_impl)
743
+ });
744
+}
csrc/cpu/cache.cpp
new
+139
@@ -0,0 +1,139 @@
1
+#include <map>
2
+#include <vector>
3
+
4
+#include "cpu_types.hpp"
5
+
6
+namespace {
7
+template <typename scalar_t>
8
+void copy_blocks_cpu_impl(
9
+ std::vector<torch::Tensor> &key_caches,
10
+ std::vector<torch::Tensor> &value_caches,
11
+ const std::vector<std::pair<int64_t, int64_t>> mapping_pairs,
12
+ const int element_num_per_block, const int layer_num) {
13
+ const size_t pair_num = mapping_pairs.size();
14
+ const size_t block_bytes = sizeof(scalar_t) * element_num_per_block;
15
+#pragma omp parallel for collapse(2)
16
+ for (int layer = 0; layer < layer_num; ++layer) {
17
+ for (size_t pair = 0; pair < pair_num; ++pair) {
18
+ int64_t source_offset = element_num_per_block * mapping_pairs[pair].first;
19
+ int64_t target_offset =
20
+ element_num_per_block * mapping_pairs[pair].second;
21
+ scalar_t *key_cache_ptr = key_caches[layer].data_ptr<scalar_t>();
22
+ scalar_t *source_ptr = key_cache_ptr + source_offset;
23
+ scalar_t *target_ptr = key_cache_ptr + target_offset;
24
+ std::memcpy(target_ptr, source_ptr, block_bytes);
25
+
26
+ scalar_t *value_cache_ptr = value_caches[layer].data_ptr<scalar_t>();
27
+ source_ptr = value_cache_ptr + source_offset;
28
+ target_ptr = value_cache_ptr + target_offset;
29
+ std::memcpy(target_ptr, source_ptr, block_bytes);
30
+ }
31
+ }
32
+}
33
+
34
+template <typename scalar_t>
35
+void reshape_and_cache_cpu_impl(
36
+ const scalar_t *__restrict__ key, const scalar_t *__restrict__ value,
37
+ scalar_t *__restrict__ key_cache, scalar_t *__restrict__ value_cache,
38
+ const int64_t *__restrict__ slot_mapping, const int num_tokens,
39
+ const int key_stride, const int value_stride, const int num_heads,
40
+ const int head_size, const int block_size, const int x) {
41
+ const int block_elem_num = num_heads * head_size * block_size;
42
+
43
+#pragma omp parallel for collapse(2)
44
+ for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
45
+ for (int head_idx = 0; head_idx < num_heads; ++head_idx) {
46
+ const int64_t slot_idx = slot_mapping[token_idx];
47
+ if (slot_idx >= 0) {
48
+ int src_key_head_idx = token_idx * key_stride + head_idx * head_size;
49
+ int src_value_head_idx =
50
+ token_idx * value_stride + head_idx * head_size;
51
+ const scalar_t *src_key_head_ptr = key + src_key_head_idx;
52
+ const scalar_t *src_value_head_ptr = value + src_value_head_idx;
53
+ const int64_t block_index = slot_idx / block_size;
54
+ const int64_t block_offset = slot_idx % block_size;
55
+ scalar_t *target_key_head_ptr = key_cache +
56
+ block_elem_num * block_index +
57
+ head_idx * block_size * head_size;
58
+ scalar_t *target_value_head_ptr = value_cache +
59
+ block_elem_num * block_index +
60
+ head_idx * block_size * head_size;
61
+
62
+ for (int src_key_idx = 0; src_key_idx < head_size; src_key_idx += x) {
63
+ const int64_t target_offset =
64
+ src_key_idx * block_size + block_offset * x;
65
+ for (int i = 0; i < x; ++i) {
66
+ target_key_head_ptr[target_offset + i] =
67
+ src_key_head_ptr[src_key_idx + i];
68
+ }
69
+ }
70
+
71
+ for (int src_value_idx = 0; src_value_idx < head_size;
72
+ ++src_value_idx) {
73
+ const int64_t target_offset =
74
+ src_value_idx * block_size + block_offset;
75
+ target_value_head_ptr[target_offset] =
76
+ src_value_head_ptr[src_value_idx];
77
+ }
78
+ }
79
+ }
80
+ }
81
+}
82
+}; // namespace
83
+
84
+void copy_blocks(std::vector<torch::Tensor> &key_caches,
85
+ std::vector<torch::Tensor> &value_caches,
86
+ const std::map<int64_t, std::vector<int64_t>> &block_mapping) {
87
+ int num_layers = key_caches.size();
88
+ TORCH_CHECK(num_layers == value_caches.size());
89
+ if (num_layers == 0) {
90
+ return;
91
+ }
92
+
93
+ std::vector<std::pair<int64_t, int64_t>> mapping_pairs;
94
+ mapping_pairs.reserve(block_mapping.size());
95
+ for (const auto &pair : block_mapping) {
96
+ for (const auto &dst : pair.second) {
97
+ mapping_pairs.emplace_back(pair.first, dst);
98
+ }
99
+ }
100
+
101
+ const int element_num_per_block = key_caches[0][0].numel();
102
+ VLLM_DISPATCH_FLOATING_TYPES(
103
+ key_caches[0].scalar_type(), "copy_blocks_cpu_impl", [&] {
104
+ CPU_KERNEL_GUARD_IN(copy_blocks_cpu_impl)
105
+ copy_blocks_cpu_impl<scalar_t>(key_caches, value_caches, mapping_pairs,
106
+ element_num_per_block, num_layers);
107
+ CPU_KERNEL_GUARD_OUT(copy_blocks_cpu_impl)
108
+ });
109
+}
110
+
111
+void reshape_and_cache(torch::Tensor &key, torch::Tensor &value,
112
+ torch::Tensor &key_cache, torch::Tensor &value_cache,
113
+ torch::Tensor &slot_mapping,
114
+ const std::string &kv_cache_dtype) {
115
+ int num_tokens = key.size(0);
116
+ int num_heads = key.size(1);
117
+ int head_size = key.size(2);
118
+ int block_size = key_cache.size(3);
119
+ int x = key_cache.size(4);
120
+
121
+ int key_stride = key.stride(0);
122
+ int value_stride = value.stride(0);
123
+
124
+ VLLM_DISPATCH_FLOATING_TYPES(
125
+ key.scalar_type(), "reshape_and_cache_cpu_impl", [&] {
126
+ CPU_KERNEL_GUARD_IN(reshape_and_cache_cpu_impl)
127
+ reshape_and_cache_cpu_impl<scalar_t>(
128
+ key.data_ptr<scalar_t>(), value.data_ptr<scalar_t>(),
129
+ key_cache.data_ptr<scalar_t>(), value_cache.data_ptr<scalar_t>(),
130
+ slot_mapping.data_ptr<int64_t>(), num_tokens, key_stride,
131
+ value_stride, num_heads, head_size, block_size, x);
132
+ CPU_KERNEL_GUARD_OUT(reshape_and_cache_cpu_impl)
133
+ });
134
+}
135
+
136
+void swap_blocks(torch::Tensor &src, torch::Tensor &dst,
137
+ const std::map<int64_t, int64_t> &block_mapping) {
138
+ TORCH_CHECK(false, "swap_blocks is unsupported on CPU.")
139
+}
csrc/cpu/cpu_types.hpp
new
+352
@@ -0,0 +1,352 @@
1
+
2
+#ifndef CPU_TYPES_HPP
3
+#define CPU_TYPES_HPP
4
+
5
+#include <immintrin.h>
6
+#include <torch/extension.h>
7
+
8
+namespace vec_op {
9
+
10
+// FIXME: FP16 is not fully supported in Torch-CPU
11
+#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
12
+ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
13
+ AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
14
+
15
+#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
16
+ AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
17
+
18
+#ifndef CPU_OP_GUARD
19
+#define CPU_KERNEL_GUARD_IN(NAME)
20
+#define CPU_KERNEL_GUARD_OUT(NAME)
21
+#else
22
+#define CPU_KERNEL_GUARD_IN(NAME) \
23
+ std::cout << #NAME << " invoked." << std::endl;
24
+#define CPU_KERNEL_GUARD_OUT(NAME) std::cout << #NAME << " exit." << std::endl;
25
+#endif
26
+
27
+#define FORCE_INLINE __attribute__((always_inline)) inline
28
+
29
+namespace {
30
+template <typename T, T... indexes, typename F>
31
+constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F &&f) {
32
+ (f(std::integral_constant<T, indexes>{}), ...);
33
+}
34
+}; // namespace
35
+
36
+template <typename T, T count, typename F,
37
+ typename = std::enable_if_t<std::is_invocable_v<F, T>>>
38
+constexpr void unroll_loop(F &&f) {
39
+ unroll_loop_item(std::make_integer_sequence<T, count>{}, std::forward<F>(f));
40
+}
41
+
42
+template <typename T> struct Vec {
43
+ constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; }
44
+};
45
+
46
+struct FP32Vec8;
47
+struct FP32Vec16;
48
+
49
+#ifdef __AVX512FP16__
50
+struct FP16Vec8 : public Vec<FP16Vec8> {
51
+ constexpr static int VEC_ELEM_NUM = 8;
52
+
53
+ __m128h reg;
54
+
55
+ explicit FP16Vec8(_Float16 v) : reg(_mm_set1_ph(v)) {}
56
+
57
+ explicit FP16Vec8(const void *ptr) : reg(_mm_loadu_ph(ptr)) {}
58
+
59
+ explicit FP16Vec8(__m128h data) : reg(data) {}
60
+
61
+ FP16Vec8 operator*(const FP16Vec8 &b) const {
62
+ return FP16Vec8(_mm_mul_ph(reg, b.reg));
63
+ }
64
+
65
+ FP16Vec8 operator+(const FP16Vec8 &b) const {
66
+ return FP16Vec8(_mm_add_ph(reg, b.reg));
67
+ }
68
+
69
+ FP16Vec8 operator-(const FP16Vec8 &b) const {
70
+ return FP16Vec8(_mm_sub_ph(reg, b.reg));
71
+ }
72
+
73
+ FP16Vec8 operator/(const FP16Vec8 &b) const {
74
+ return FP16Vec8(_mm_div_ph(reg, b.reg));
75
+ }
76
+
77
+ void save(void *ptr) const { _mm_storeu_ph(ptr, reg); }
78
+};
79
+#endif
80
+
81
+struct BF16Vec8 : public Vec<BF16Vec8> {
82
+ constexpr static int VEC_ELEM_NUM = 8;
83
+
84
+ __m128i reg;
85
+
86
+ explicit BF16Vec8(const void *ptr)
87
+ : reg((__m128i)_mm_loadu_si128((__m128i *)ptr)) {}
88
+
89
+ explicit BF16Vec8(const FP32Vec8 &);
90
+
91
+ void save(void *ptr) const { *reinterpret_cast<__m128i *>(ptr) = reg; }
92
+};
93
+
94
+struct BF16Vec16 : public Vec<BF16Vec16> {
95
+ constexpr static int VEC_ELEM_NUM = 16;
96
+
97
+ __m256i reg;
98
+
99
+ explicit BF16Vec16(const void *ptr)
100
+ : reg((__m256i)_mm256_loadu_si256((__m256i *)ptr)) {}
101
+
102
+ explicit BF16Vec16(const FP32Vec16 &);
103
+
104
+ void save(void *ptr) const { *reinterpret_cast<__m256i *>(ptr) = reg; }
105
+};
106
+
107
+struct BF16Vec32 : public Vec<BF16Vec32> {
108
+ constexpr static int VEC_ELEM_NUM = 32;
109
+
110
+ __m512i reg;
111
+
112
+ explicit BF16Vec32(const void *ptr) : reg((__m512i)_mm512_loadu_si512(ptr)) {}
113
+
114
+ explicit BF16Vec32(__m512i data) : reg(data) {}
115
+
116
+ explicit BF16Vec32(BF16Vec8 &vec8_data)
117
+ : reg((__m512i)_mm512_inserti32x4(
118
+ _mm512_inserti32x4(_mm512_inserti32x4(_mm512_castsi128_si512(
119
+ (__m128i)vec8_data.reg),
120
+ (__m128i)vec8_data.reg, 1),
121
+ (__m128i)vec8_data.reg, 2),
122
+ (__m128i)vec8_data.reg, 3)) {}
123
+
124
+ void save(void *ptr) const { *reinterpret_cast<__m512i *>(ptr) = reg; }
125
+};
126
+
127
+struct FP32Vec4 : public Vec<FP32Vec4> {
128
+ constexpr static int VEC_ELEM_NUM = 4;
129
+ union AliasReg {
130
+ __m128 reg;
131
+ float values[VEC_ELEM_NUM];
132
+ };
133
+
134
+ __m128 reg;
135
+
136
+ explicit FP32Vec4(float v) : reg(_mm_set1_ps(v)) {}
137
+
138
+ explicit FP32Vec4() : reg(_mm_set1_ps(0.0)) {}
139
+
140
+ explicit FP32Vec4(const float *ptr) : reg(_mm_loadu_ps(ptr)) {}
141
+
142
+ explicit FP32Vec4(__m128 data) : reg(data) {}
143
+
144
+ explicit FP32Vec4(const FP32Vec4 &data) : reg(data.reg) {}
145
+};
146
+
147
+struct FP32Vec8 : public Vec<FP32Vec8> {
148
+ constexpr static int VEC_ELEM_NUM = 8;
149
+ union AliasReg {
150
+ __m256 reg;
151
+ float values[VEC_ELEM_NUM];
152
+ };
153
+
154
+ __m256 reg;
155
+
156
+ explicit FP32Vec8(float v) : reg(_mm256_set1_ps(v)) {}
157
+
158
+ explicit FP32Vec8() : reg(_mm256_set1_ps(0.0)) {}
159
+
160
+ explicit FP32Vec8(const float *ptr) : reg(_mm256_loadu_ps(ptr)) {}
161
+
162
+ explicit FP32Vec8(__m256 data) : reg(data) {}
163
+
164
+ explicit FP32Vec8(const FP32Vec8 &data) : reg(data.reg) {}
165
+
166
+#ifdef __AVX512FP16__
167
+ explicit FP32Vec8(__m128h v) : reg(_mm256_cvtph_ps(_mm_castph_si128(v))) {}
168
+#endif
169
+
170
+ explicit FP32Vec8(const BF16Vec8 &v)
171
+ : reg(_mm256_castsi256_ps(
172
+ _mm256_bslli_epi128(_mm256_cvtepu16_epi32(v.reg), 2))) {}
173
+
174
+ float reduce_sum() const {
175
+ AliasReg ar;
176
+ ar.reg = reg;
177
+ float result = 0;
178
+ unroll_loop<int, VEC_ELEM_NUM>([&result, &ar](int i) { result += ar.values[i]; });
179
+
180
+ return result;
181
+ }
182
+
183
+ FP32Vec8 exp() const {
184
+ AliasReg ar;
185
+ ar.reg = reg;
186
+ return FP32Vec8(_mm256_set_ps(expf(ar.values[7]), expf(ar.values[6]),
187
+ expf(ar.values[5]), expf(ar.values[4]),
188
+ expf(ar.values[3]), expf(ar.values[2]),
189
+ expf(ar.values[1]), expf(ar.values[0])));
190
+ }
191
+
192
+ FP32Vec8 tanh() const {
193
+ AliasReg ar;
194
+ ar.reg = reg;
195
+ return FP32Vec8(_mm256_set_ps(tanhf(ar.values[7]), tanhf(ar.values[6]),
196
+ tanhf(ar.values[5]), tanhf(ar.values[4]),
197
+ tanhf(ar.values[3]), tanhf(ar.values[2]),
198
+ tanhf(ar.values[1]), tanhf(ar.values[0])));
199
+ }
200
+
201
+ FP32Vec8 er() const {
202
+ AliasReg ar;
203
+ ar.reg = reg;
204
+ return FP32Vec8(_mm256_set_ps(erf(ar.values[7]), erf(ar.values[6]),
205
+ erf(ar.values[5]), erf(ar.values[4]),
206
+ erf(ar.values[3]), erf(ar.values[2]),
207
+ erf(ar.values[1]), erf(ar.values[0])));
208
+ }
209
+
210
+ FP32Vec8 operator*(const FP32Vec8 &b) const {
211
+ return FP32Vec8(_mm256_mul_ps(reg, b.reg));
212
+ }
213
+
214
+ FP32Vec8 operator+(const FP32Vec8 &b) const {
215
+ return FP32Vec8(_mm256_add_ps(reg, b.reg));
216
+ }
217
+
218
+ FP32Vec8 operator-(const FP32Vec8 &b) const {
219
+ return FP32Vec8(_mm256_sub_ps(reg, b.reg));
220
+ }
221
+
222
+ FP32Vec8 operator/(const FP32Vec8 &b) const {
223
+ return FP32Vec8(_mm256_div_ps(reg, b.reg));
224
+ }
225
+
226
+ void save(float *ptr) const { _mm256_storeu_ps(ptr, reg); }
227
+};
228
+
229
+struct FP32Vec16 : public Vec<FP32Vec16> {
230
+ constexpr static int VEC_ELEM_NUM = 16;
231
+ union AliasReg {
232
+ __m512 reg;
233
+ float values[VEC_ELEM_NUM];
234
+ };
235
+
236
+ __m512 reg;
237
+
238
+ explicit FP32Vec16(float v) : reg(_mm512_set1_ps(v)) {}
239
+
240
+ explicit FP32Vec16() : reg(_mm512_set1_ps(0.0)) {}
241
+
242
+ explicit FP32Vec16(const float *ptr) : reg(_mm512_loadu_ps(ptr)) {}
243
+
244
+ explicit FP32Vec16(__m512 data) : reg(data) {}
245
+
246
+ explicit FP32Vec16(const FP32Vec16 &data) : reg(data.reg) {}
247
+
248
+ explicit FP32Vec16(const FP32Vec4 &data)
249
+ : reg((__m512)_mm512_inserti32x4(
250
+ _mm512_inserti32x4(
251
+ _mm512_inserti32x4(_mm512_castsi128_si512((__m128i)data.reg),
252
+ (__m128i)data.reg, 1),
253
+ (__m128i)data.reg, 2),
254
+ (__m128i)data.reg, 3)) {}
255
+
256
+ explicit FP32Vec16(const FP32Vec8 &data)
257
+ : reg((__m512)_mm512_inserti32x8(
258
+ _mm512_castsi256_si512((__m256i)data.reg), (__m256i)data.reg, 1)) {}
259
+
260
+ explicit FP32Vec16(const BF16Vec16 &v)
261
+ : reg(_mm512_castsi512_ps(
262
+ _mm512_bslli_epi128(_mm512_cvtepu16_epi32(v.reg), 2))) {}
263
+
264
+ explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
265
+
266
+ FP32Vec16 operator*(const FP32Vec16 &b) const {
267
+ return FP32Vec16(_mm512_mul_ps(reg, b.reg));
268
+ }
269
+
270
+ FP32Vec16 operator+(const FP32Vec16 &b) const {
271
+ return FP32Vec16(_mm512_add_ps(reg, b.reg));
272
+ }
273
+
274
+ FP32Vec16 operator-(const FP32Vec16 &b) const {
275
+ return FP32Vec16(_mm512_sub_ps(reg, b.reg));
276
+ }
277
+
278
+ FP32Vec16 operator/(const FP32Vec16 &b) const {
279
+ return FP32Vec16(_mm512_div_ps(reg, b.reg));
280
+ }
281
+
282
+ float reduce_sum() const { return _mm512_reduce_add_ps(reg); }
283
+
284
+ template <int group_size> float reduce_sub_sum(int idx) {
285
+ static_assert(VEC_ELEM_NUM % group_size == 0);
286
+ constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
287
+ __mmask16 mask = _cvtu32_mask16(base_mask << (idx * group_size));
288
+ return _mm512_mask_reduce_add_ps(mask, reg);
289
+ }
290
+
291
+ void save(float *ptr) const { _mm512_storeu_ps(ptr, reg); }
292
+};
293
+
294
+template <typename T> struct VecType { using vec_type = void; };
295
+
296
+template <typename T> using vec_t = typename VecType<T>::vec_type;
297
+
298
+template <> struct VecType<float> { using vec_type = FP32Vec8; };
299
+
300
+#ifdef __AVX512FP16__
301
+template <> struct VecType<c10::Half> { using vec_type = FP16Vec16; };
302
+#endif
303
+
304
+template <> struct VecType<c10::BFloat16> { using vec_type = BF16Vec8; };
305
+
306
+template <typename T> void storeFP32(float v, T *ptr) { *ptr = v; }
307
+
308
+#ifdef __AVX512FP16__
309
+template <> inline void storeFP32<c10::Half>(float v, c10::Half *ptr) {
310
+ *reinterpret_cast<_Float16 *>(ptr) = v;
311
+}
312
+#endif
313
+
314
+inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
315
+ acc = acc + a * b;
316
+}
317
+
318
+#ifdef __AVX512BF16__
319
+template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
320
+ *reinterpret_cast<__bfloat16 *>(ptr) = _mm_cvtness_sbh(v);
321
+}
322
+
323
+inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
324
+ : reg((__m128i)_mm256_cvtneps_pbh(v.reg)) {}
325
+
326
+inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
327
+ : reg((__m256i)_mm512_cvtneps_pbh(v.reg)) {}
328
+
329
+inline void fma(FP32Vec16 &acc, BF16Vec32 &a, BF16Vec32 &b) {
330
+ acc.reg = _mm512_dpbf16_ps(acc.reg, (__m512bh)a.reg, (__m512bh)b.reg);
331
+}
332
+#else
333
+template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
334
+ c10::BFloat16 __attribute__((__may_alias__)) *v_ptr =
335
+ reinterpret_cast<c10::BFloat16 *>(&v);
336
+ *ptr = *(v_ptr + 1);
337
+}
338
+
339
+inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
340
+ : reg(_mm256_cvtepi32_epi16(
341
+ _mm256_bsrli_epi128(_mm256_castps_si256(v.reg), 2))) {}
342
+
343
+inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
344
+ : reg(_mm512_cvtepi32_epi16(
345
+ _mm512_bsrli_epi128(_mm512_castps_si512(v.reg), 2))) {}
346
+#endif
347
+
348
+inline void prefetch(const void *addr) { _mm_prefetch(addr, _MM_HINT_T1); }
349
+
350
+}; // namespace vec_op
351
+
352
+#endif
csrc/cpu/layernorm.cpp
new
+117
@@ -0,0 +1,117 @@
1
+#include "cpu_types.hpp"
2
+
3
+namespace {
4
+template <typename scalar_t>
5
+void rms_norm_impl(scalar_t *__restrict__ out,
6
+ const scalar_t *__restrict__ input,
7
+ const scalar_t *__restrict__ weight, const float epsilon,
8
+ const int num_tokens, const int hidden_size) {
9
+ using scalar_vec_t = vec_op::vec_t<scalar_t>;
10
+ constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
11
+ TORCH_CHECK(hidden_size % VEC_ELEM_NUM == 0);
12
+
13
+#pragma omp parallel for
14
+ for (int i = 0; i < num_tokens; ++i) {
15
+ vec_op::FP32Vec8 variance(0.0);
16
+ auto input_p = input + i * hidden_size;
17
+ auto output_p = out + i * hidden_size;
18
+ for (int j = 0; j < hidden_size; j += VEC_ELEM_NUM) {
19
+ scalar_vec_t x(input_p + j);
20
+ vec_op::FP32Vec8 fp32_x(x);
21
+ variance = variance + fp32_x * fp32_x;
22
+ }
23
+
24
+ float s_variance =
25
+ 1.0f / sqrtf(variance.reduce_sum() / (float)hidden_size + epsilon);
26
+ vec_op::FP32Vec8 fp32_s_variance(s_variance);
27
+
28
+ for (int j = 0; j < hidden_size; j += VEC_ELEM_NUM) {
29
+ scalar_vec_t x(input_p + j);
30
+ scalar_vec_t w(weight + j);
31
+
32
+ vec_op::FP32Vec8 fp32_x(x);
33
+ vec_op::FP32Vec8 fp32_w(w);
34
+
35
+ vec_op::FP32Vec8 fp32_out = fp32_x * fp32_s_variance * fp32_w;
36
+
37
+ scalar_vec_t out(fp32_out);
38
+ out.save(output_p + j);
39
+ }
40
+ }
41
+}
42
+
43
+template <typename scalar_t>
44
+void fused_add_rms_norm_impl(scalar_t *__restrict__ input,
45
+ scalar_t *__restrict__ residual,
46
+ const scalar_t *__restrict__ weight,
47
+ const float epsilon, const int num_tokens,
48
+ const int hidden_size) {
49
+ using scalar_vec_t = vec_op::vec_t<scalar_t>;
50
+ constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
51
+ TORCH_CHECK(hidden_size % VEC_ELEM_NUM == 0);
52
+
53
+#pragma omp parallel for
54
+ for (int i = 0; i < num_tokens; ++i) {
55
+ vec_op::FP32Vec8 variance(0.0);
56
+ auto input_p = input + i * hidden_size;
57
+ auto residual_p = residual + i * hidden_size;
58
+ for (int j = 0; j < hidden_size; j += VEC_ELEM_NUM) {
59
+ scalar_vec_t x(input_p + j);
60
+ scalar_vec_t res(residual_p + j);
61
+ vec_op::FP32Vec8 fp32_x(x);
62
+ vec_op::FP32Vec8 fp32_res(res);
63
+
64
+ fp32_x = fp32_x + fp32_res;
65
+ variance = variance + fp32_x * fp32_x;
66
+ scalar_vec_t out(fp32_x);
67
+ out.save(residual_p + j);
68
+ }
69
+
70
+ float s_variance =
71
+ 1.0f / sqrtf(variance.reduce_sum() / (float)hidden_size + epsilon);
72
+ vec_op::FP32Vec8 fp32_s_variance(s_variance);
73
+
74
+ for (int j = 0; j < hidden_size; j += VEC_ELEM_NUM) {
75
+ scalar_vec_t w(weight + j);
76
+ scalar_vec_t res(residual_p + j);
77
+
78
+ vec_op::FP32Vec8 fp32_w(w);
79
+ vec_op::FP32Vec8 fp32_res(res);
80
+
81
+ vec_op::FP32Vec8 fp32_out = fp32_res * fp32_s_variance * fp32_w;
82
+
83
+ scalar_vec_t out(fp32_out);
84
+ out.save(input_p + j);
85
+ }
86
+ }
87
+}
88
+} // namespace
89
+
90
+void rms_norm(torch::Tensor &out, torch::Tensor &input,
91
+ torch::Tensor &weight, float epsilon) {
92
+ int hidden_size = input.size(-1);
93
+ int num_tokens = input.numel() / hidden_size;
94
+
95
+ VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "rms_norm_impl", [&] {
96
+ CPU_KERNEL_GUARD_IN(rms_norm_impl)
97
+ rms_norm_impl(out.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
98
+ weight.data_ptr<scalar_t>(), epsilon, num_tokens,
99
+ hidden_size);
100
+ CPU_KERNEL_GUARD_OUT(rms_norm_impl)
101
+ });
102
+}
103
+
104
+void fused_add_rms_norm(torch::Tensor &input, torch::Tensor &residual,
105
+ torch::Tensor &weight, float epsilon) {
106
+ int hidden_size = input.size(-1);
107
+ int num_tokens = input.numel() / hidden_size;
108
+
109
+ VLLM_DISPATCH_FLOATING_TYPES(
110
+ input.scalar_type(), "fused_add_rms_norm_impl", [&] {
111
+ CPU_KERNEL_GUARD_IN(fused_add_rms_norm_impl)
112
+ fused_add_rms_norm_impl(
113
+ input.data_ptr<scalar_t>(), residual.data_ptr<scalar_t>(),
114
+ weight.data_ptr<scalar_t>(), epsilon, num_tokens, hidden_size);
115
+ CPU_KERNEL_GUARD_OUT(fused_add_rms_norm_impl)
116
+ });
117
+}
csrc/cpu/pos_encoding.cpp
new
+199
@@ -0,0 +1,199 @@
1
+
2
+#include "cpu_types.hpp"
3
+
4
+namespace {
5
+template <typename scalar_t>
6
+void rotary_embedding_impl(
7
+ const int64_t
8
+ *__restrict__ positions, // [batch_size, seq_len] or [num_tokens]
9
+ scalar_t
10
+ *__restrict__ query, /// [batch_size, seq_len, num_heads, head_size] or
11
+ /// [num_tokens, num_heads, head_size]
12
+ scalar_t
13
+ *__restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or
14
+ // [num_tokens, num_kv_heads, head_size]
15
+ const scalar_t
16
+ *__restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
17
+ const int rot_dim, const int64_t query_stride, const int64_t key_stride,
18
+ const int num_heads, const int num_kv_heads, const int head_size,
19
+ const int num_tokens) {
20
+ using scalar_vec_t = vec_op::vec_t<scalar_t>;
21
+ constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
22
+ constexpr int ELEM_SIZE = sizeof(scalar_t);
23
+
24
+ const int embed_dim = rot_dim / 2;
25
+ TORCH_CHECK(embed_dim % VEC_ELEM_NUM == 0);
26
+
27
+#pragma omp parallel for
28
+ for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
29
+ int64_t pos = positions[token_idx];
30
+ const scalar_t *cache_ptr = cos_sin_cache + pos * rot_dim;
31
+
32
+ for (int i = 0; i < num_heads; ++i) {
33
+ const int head_idx = i;
34
+ const int64_t token_head =
35
+ token_idx * query_stride + head_idx * head_size;
36
+ for (int j = 0; j < embed_dim; j += VEC_ELEM_NUM) {
37
+ const int rot_offset = j;
38
+ const int x_index = rot_offset;
39
+ const int y_index = embed_dim + rot_offset;
40
+
41
+ const int64_t out_x = token_head + x_index;
42
+ const int64_t out_y = token_head + y_index;
43
+
44
+ const scalar_vec_t cos(cache_ptr + x_index);
45
+ const scalar_vec_t sin(cache_ptr + y_index);
46
+
47
+ const scalar_vec_t q_x(query + out_x);
48
+ const scalar_vec_t q_y(query + out_y);
49
+
50
+ vec_op::FP32Vec8 fp32_cos(cos);
51
+ vec_op::FP32Vec8 fp32_sin(sin);
52
+
53
+ vec_op::FP32Vec8 fp32_q_x(q_x);
54
+ vec_op::FP32Vec8 fp32_q_y(q_y);
55
+
56
+ auto out1 = fp32_q_x * fp32_cos - fp32_q_y * fp32_sin;
57
+ scalar_vec_t(out1).save(query + out_x);
58
+
59
+ auto out2 = fp32_q_y * fp32_cos + fp32_q_x * fp32_sin;
60
+ scalar_vec_t(out2).save(query + out_y);
61
+ }
62
+ }
63
+
64
+ for (int i = 0; i < num_kv_heads; ++i) {
65
+ const int head_idx = i;
66
+ const int64_t token_head = token_idx * key_stride + head_idx * head_size;
67
+ for (int j = 0; j < embed_dim; j += VEC_ELEM_NUM) {
68
+ const int rot_offset = j;
69
+ const int x_index = rot_offset;
70
+ const int y_index = embed_dim + rot_offset;
71
+
72
+ const int64_t out_x = token_head + x_index;
73
+ const int64_t out_y = token_head + y_index;
74
+
75
+ const scalar_vec_t cos(cache_ptr + x_index);
76
+ const scalar_vec_t sin(cache_ptr + y_index);
77
+
78
+ const scalar_vec_t k_x(key + out_x);
79
+ const scalar_vec_t k_y(key + out_y);
80
+
81
+ vec_op::FP32Vec8 fp32_cos(cos);
82
+ vec_op::FP32Vec8 fp32_sin(sin);
83
+
84
+ vec_op::FP32Vec8 fp32_k_x(k_x);
85
+ vec_op::FP32Vec8 fp32_k_y(k_y);
86
+
87
+ auto out1 = fp32_k_x * fp32_cos - fp32_k_y * fp32_sin;
88
+ scalar_vec_t(out1).save(key + out_x);
89
+ auto out2 = fp32_k_y * fp32_cos + fp32_k_x * fp32_sin;
90
+ scalar_vec_t(out2).save(key + out_y);
91
+ }
92
+ }
93
+ }
94
+}
95
+
96
+template <typename scalar_t>
97
+void rotary_embedding_gptj_impl(
98
+ const int64_t
99
+ *__restrict__ positions, // [batch_size, seq_len] or [num_tokens]
100
+ scalar_t
101
+ *__restrict__ query, /// [batch_size, seq_len, num_heads, head_size] or
102
+ /// [num_tokens, num_heads, head_size]
103
+ scalar_t
104
+ *__restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or
105
+ // [num_tokens, num_kv_heads, head_size]
106
+ const scalar_t
107
+ *__restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
108
+ const int rot_dim, const int64_t query_stride, const int64_t key_stride,
109
+ const int num_heads, const int num_kv_heads, const int head_size,
110
+ const int num_tokens) {
111
+ const int embed_dim = rot_dim / 2;
112
+
113
+#pragma omp parallel for collapse(2)
114
+ for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
115
+ for (int i = 0; i < num_heads; ++i) {
116
+ int64_t pos = positions[token_idx];
117
+ const scalar_t *cache_ptr = cos_sin_cache + pos * rot_dim;
118
+ const scalar_t *cos_cache_ptr = cache_ptr;
119
+ const scalar_t *sin_cache_ptr = cache_ptr + embed_dim;
120
+ const int head_idx = i;
121
+ const int64_t token_head =
122
+ token_idx * query_stride + head_idx * head_size;
123
+ scalar_t *head_query = token_head + query;
124
+ for (int j = 0; j < embed_dim; j += 1) {
125
+ const int rot_offset = j;
126
+ const int x_index = 2 * rot_offset;
127
+ const int y_index = 2 * rot_offset + 1;
128
+
129
+ const float cos = cos_cache_ptr[rot_offset];
130
+ const float sin = sin_cache_ptr[rot_offset];
131
+
132
+ const float x = head_query[x_index];
133
+ const float y = head_query[y_index];
134
+
135
+ head_query[x_index] = x * cos - y * sin;
136
+ head_query[y_index] = y * cos + x * sin;
137
+ }
138
+ }
139
+ }
140
+
141
+#pragma omp parallel for collapse(2)
142
+ for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
143
+ for (int i = 0; i < num_kv_heads; ++i) {
144
+ int64_t pos = positions[token_idx];
145
+ const scalar_t *cache_ptr = cos_sin_cache + pos * rot_dim;
146
+ const scalar_t *cos_cache_ptr = cache_ptr;
147
+ const scalar_t *sin_cache_ptr = cache_ptr + embed_dim;
148
+ const int head_idx = i;
149
+ const int64_t token_head = token_idx * key_stride + head_idx * head_size;
150
+ scalar_t *head_key = key + token_head;
151
+ for (int j = 0; j < embed_dim; j += 1) {
152
+ const int rot_offset = j;
153
+ const int x_index = 2 * rot_offset;
154
+ const int y_index = 2 * rot_offset + 1;
155
+
156
+ const float cos = cos_cache_ptr[rot_offset];
157
+ const float sin = sin_cache_ptr[rot_offset];
158
+
159
+ const float x = head_key[x_index];
160
+ const float y = head_key[y_index];
161
+
162
+ head_key[x_index] = x * cos - y * sin;
163
+ head_key[y_index] = y * cos + x * sin;
164
+ }
165
+ }
166
+ }
167
+}
168
+}; // namespace
169
+
170
+void rotary_embedding(torch::Tensor &positions, torch::Tensor &query,
171
+ torch::Tensor &key, int head_size,
172
+ torch::Tensor &cos_sin_cache, bool is_neox) {
173
+ int num_tokens = query.numel() / query.size(-1);
174
+ int rot_dim = cos_sin_cache.size(1);
175
+ int num_heads = query.size(-1) / head_size;
176
+ int num_kv_heads = key.size(-1) / head_size;
177
+ int64_t key_stride = key.stride(-2);
178
+ int64_t query_stride = query.stride(-2);
179
+
180
+ VLLM_DISPATCH_FLOATING_TYPES(
181
+ query.scalar_type(), "rotary_embedding_impl", [&] {
182
+ CPU_KERNEL_GUARD_IN(rotary_embedding_impl)
183
+ if (is_neox) {
184
+ rotary_embedding_impl(
185
+ positions.data_ptr<int64_t>(), query.data_ptr<scalar_t>(),
186
+ key.data_ptr<scalar_t>(), cos_sin_cache.data_ptr<scalar_t>(),
187
+ rot_dim, query_stride, key_stride, num_heads, num_kv_heads,
188
+ head_size, num_tokens);
189
+ } else {
190
+ rotary_embedding_gptj_impl(
191
+ positions.data_ptr<int64_t>(), query.data_ptr<scalar_t>(),
192
+ key.data_ptr<scalar_t>(), cos_sin_cache.data_ptr<scalar_t>(),
193
+ rot_dim, query_stride, key_stride, num_heads, num_kv_heads,
194
+ head_size, num_tokens);
195
+ }
196
+
197
+ CPU_KERNEL_GUARD_OUT(rotary_embedding_impl)
198
+ });
199
+}
csrc/cpu/pybind.cpp
new
+73
@@ -0,0 +1,73 @@
1
+#include "cache.h"
2
+#include "cuda_utils.h"
3
+#include "ops.h"
4
+#include <torch/extension.h>
5
+
6
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
7
+ // vLLM custom ops
8
+ pybind11::module ops = m.def_submodule("ops", "vLLM custom operators");
9
+
10
+ // Attention ops
11
+ ops.def(
12
+ "paged_attention_v1",
13
+ &paged_attention_v1,
14
+ "Compute the attention between an input query and the cached keys/values using PagedAttention.");
15
+ ops.def(
16
+ "paged_attention_v2",
17
+ &paged_attention_v2,
18
+ "PagedAttention V2.");
19
+
20
+ // Activation ops
21
+ ops.def(
22
+ "silu_and_mul",
23
+ &silu_and_mul,
24
+ "Activation function used in SwiGLU.");
25
+ ops.def(
26
+ "gelu_and_mul",
27
+ &gelu_and_mul,
28
+ "Activation function used in GeGLU with `none` approximation.");
29
+ ops.def(
30
+ "gelu_tanh_and_mul",
31
+ &gelu_tanh_and_mul,
32
+ "Activation function used in GeGLU with `tanh` approximation.");
33
+ ops.def(
34
+ "gelu_new",
35
+ &gelu_new,
36
+ "GELU implementation used in GPT-2.");
37
+ ops.def(
38
+ "gelu_fast",
39
+ &gelu_fast,
40
+ "Approximate GELU implementation.");
41
+
42
+ // Layernorm
43
+ ops.def(
44
+ "rms_norm",
45
+ &rms_norm,
46
+ "Apply Root Mean Square (RMS) Normalization to the input tensor.");
47
+
48
+ ops.def(
49
+ "fused_add_rms_norm",
50
+ &fused_add_rms_norm,
51
+ "In-place fused Add and RMS Normalization");
52
+
53
+ // Rotary embedding
54
+ ops.def(
55
+ "rotary_embedding",
56
+ &rotary_embedding,
57
+ "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
58
+
59
+ // Cache ops
60
+ pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
61
+ cache_ops.def(
62
+ "swap_blocks",
63
+ &swap_blocks,
64
+ "Swap in (out) the cache blocks from src to dst");
65
+ cache_ops.def(
66
+ "copy_blocks",
67
+ ©_blocks,
68
+ "Copy the cache blocks from src to dst");
69
+ cache_ops.def(
70
+ "reshape_and_cache",
71
+ &reshape_and_cache,
72
+ "Reshape the key and value tensors and cache them");
73
+}
docs/source/getting_started/cpu-installation.rst
new
+87
@@ -0,0 +1,87 @@
1
+.. _installation_cpu:
2
+
3
+Installation with CPU
4
+========================
5
+
6
+vLLM initially supports basic model inferencing and serving on x86 CPU platform, with data types FP32 and BF16.
7
+
8
+Table of contents:
9
+
10
+#. :ref:`Requirements <cpu_backend_requirements>`
11
+#. :ref:`Quick start using Dockerfile <cpu_backend_quick_start_dockerfile>`
12
+#. :ref:`Build from source <build_cpu_backend_from_source>`
13
+#. :ref:`Performance tips <cpu_backend_performance_tips>`
14
+
15
+.. _cpu_backend_requirements:
16
+
17
+Requirements
18
+------------
19
+
20
+* OS: Linux
21
+* Compiler: gcc/g++>=12.3.0 (recommended)
22
+* Instruction set architecture (ISA) requirement: AVX512 is required.
23
+
24
+.. _cpu_backend_quick_start_dockerfile:
25
+
26
+Quick start using Dockerfile
27
+----------------------------
28
+
29
+.. code-block:: console
30
+
31
+ $ docker build -f Dockerfile.cpu -t vllm-cpu-env --shm-size=4g .
32
+ $ docker run -it \
33
+ --rm \
34
+ --network=host \
35
+ --cpuset-cpus=<cpu-id-list, optional> \
36
+ --cpuset-mems=<memory-node, optional> \
37
+ vllm-cpu-env
38
+
39
+.. _build_cpu_backend_from_source:
40
+
41
+Build from source
42
+-----------------
43
+
44
+- First, install required compiler. We recommend to use ``gcc/g++ >= 12.3.0`` as the default compiler to avoid potential problems. For example, on Ubuntu 22.4, you can run:
45
+
46
+.. code-block:: console
47
+
48
+ $ sudo apt-get update -y
49
+ $ sudo apt-get install -y gcc-12 g++-12
50
+ $ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
51
+
52
+- Second, install Python packages for vLLM CPU backend building:
53
+
54
+.. code-block:: console
55
+
56
+ $ pip install --upgrade pip
57
+ $ pip install wheel packaging ninja setuptools>=49.4.0 numpy
58
+ $ pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu
59
+
60
+- Finally, build and install vLLM CPU backend:
61
+
62
+.. code-block:: console
63
+
64
+ $ VLLM_TARGET_DEVICE=cpu python setup.py install
65
+
66
+.. note::
67
+ - BF16 is the default data type in the current CPU backend (that means the backend will cast FP16 to BF16), and is compatible will all CPUs with AVX512 ISA support.
68
+
69
+ - AVX512_BF16 is an extension ISA provides native BF16 data type conversion and vector product instructions, will brings some performance improvement compared with pure AVX512. The CPU backend build script will check the host CPU flags to determine whether to enable AVX512_BF16.
70
+
71
+ - If you want to force enable AVX512_BF16 for the cross-compilation, please set environment variable VLLM_CPU_AVX512BF16=1 before the building.
72
+
73
+.. _cpu_backend_performance_tips:
74
+
75
+Performance tips
76
+-----------------
77
+
78
+- vLLM CPU backend uses environment variable ``VLLM_CPU_KVCACHE_SPACE`` to specify the KV Cache size (e.g, ``VLLM_CPU_KVCACHE_SPACE=40`` means 40 GB space for KV cache), larger setting will allow vLLM running more requests in parallel. This parameter should be set based on the hardware configuration and memory management pattern of users.
79
+
80
+- vLLM CPU backend uses OpenMP for thread-parallel computation. If you want the best performance on CPU, it will be very critical to isolate CPU cores for OpenMP threads with other thread pools (like web-service event-loop), to avoid CPU oversubscription.
81
+
82
+- If using vLLM CPU backend on a bare-metal machine, it is recommended to disable the hyper-threading.
83
+
84
+- If using vLLM CPU backend on a multi-socket machine with NUMA, be aware to set CPU cores and memory nodes, to avoid the remote memory node access. ``numactl`` is an useful tool for CPU core and memory binding on NUMA platform. Besides, ``--cpuset-cpus`` and ``--cpuset-mems`` arguments of ``docker run`` are also useful.
85
+
86
+
87
+
docs/source/index.rst
+1
@@ -63,6 +63,7 @@ Documentation
63
getting_started/installation
64
getting_started/amd-installation
65
getting_started/neuron-installation
66
+ getting_started/cpu-installation
67
getting_started/quickstart
68
69
.. toctree::
requirements-cpu.txt
new
+15
@@ -0,0 +1,15 @@
1
+cmake>=3.21
2
+ninja # For faster builds.
3
+psutil
4
+ray >= 2.9
5
+sentencepiece # Required for LLaMA tokenizer.
6
+numpy
7
+transformers >= 4.38.0 # Required for Gemma.
8
+fastapi
9
+uvicorn[standard]
10
+pydantic >= 2.0 # Required for OpenAI server.
11
+prometheus_client >= 0.18.0
12
+torch == 2.1.2+cpu
13
+triton >= 2.1.0
14
+filelock == 3.13.3
15
+py-cpuinfo
\ No newline at end of file
setup.py
+17
-2
@@ -15,6 +15,8 @@ from torch.utils.cpp_extension import CUDA_HOME
15
16
ROOT_DIR = os.path.dirname(__file__)
17
logger = logging.getLogger(__name__)
18
+# Target device of vLLM, supporting [cuda (by default), rocm, neuron, cpu]
19
+VLLM_TARGET_DEVICE = os.getenv("VLLM_TARGET_DEVICE", "cuda")
20
21
# vLLM only supports Linux platform
22
assert sys.platform.startswith(
@@ -112,6 +114,7 @@ class cmake_build_ext(build_ext):
114
'-DCMAKE_BUILD_TYPE={}'.format(cfg),
115
'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}'.format(outdir),
116
'-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}'.format(self.build_temp),
117
+ '-DVLLM_TARGET_DEVICE={}'.format(VLLM_TARGET_DEVICE),
118
]
119
120
verbose = bool(int(os.getenv('VERBOSE', '0')))
@@ -185,11 +188,14 @@ class cmake_build_ext(build_ext):
188
189
190
def _is_cuda() -> bool:
188
- return torch.version.cuda is not None and not _is_neuron()
191
+ return VLLM_TARGET_DEVICE == "cuda" \
192
+ and torch.version.cuda is not None \
193
+ and not _is_neuron()
194
195
196
def _is_hip() -> bool:
192
- return torch.version.hip is not None
197
+ return (VLLM_TARGET_DEVICE == "cuda"
198
+ or VLLM_TARGET_DEVICE == "rocm") and torch.version.hip is not None
199
200
201
def _is_neuron() -> bool:
@@ -201,6 +207,10 @@ def _is_neuron() -> bool:
207
return torch_neuronx_installed
208
209
210
+def _is_cpu() -> bool:
211
+ return VLLM_TARGET_DEVICE == "cpu"
212
+
213
+
214
def _install_punica() -> bool:
215
return bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "0")))
216
@@ -296,6 +306,8 @@ def get_vllm_version() -> str:
306
if neuron_version != MAIN_CUDA_VERSION:
307
neuron_version_str = neuron_version.replace(".", "")[:3]
308
version += f"+neuron{neuron_version_str}"
309
+ elif _is_cpu():
310
+ version += "+cpu"
311
else:
312
raise RuntimeError("Unknown runtime environment")
313
@@ -322,6 +334,9 @@ def get_requirements() -> List[str]:
334
elif _is_neuron():
335
with open(get_path("requirements-neuron.txt")) as f:
336
requirements = f.read().strip().split("\n")
337
+ elif _is_cpu():
338
+ with open(get_path("requirements-cpu.txt")) as f:
339
+ requirements = f.read().strip().split("\n")
340
else:
341
raise ValueError(
342
"Unsupported platform, please use CUDA, ROCM or Neuron.")
vllm/attention/backends/torch_sdpa.py
new
+253
@@ -0,0 +1,253 @@
1
+""" Attention layer with torch scaled_dot_product_attention
2
+ and PagedAttention."""
3
+from dataclasses import dataclass
4
+from typing import Dict, List, Optional, Tuple, Type
5
+
6
+import torch
7
+from torch.nn.functional import scaled_dot_product_attention
8
+
9
+from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
+ AttentionMetadata)
11
+from vllm.attention.ops.paged_attn import (PagedAttention,
12
+ PagedAttentionMetadata)
13
+
14
+
15
+class TorchSDPABackend(AttentionBackend):
16
+
17
+ @staticmethod
18
+ def get_impl_cls() -> Type["TorchSDPABackendImpl"]:
19
+ return TorchSDPABackendImpl
20
+
21
+ @staticmethod
22
+ def make_metadata(*args, **kwargs) -> "TorchSDPAMetadata":
23
+ return TorchSDPAMetadata(*args, **kwargs)
24
+
25
+ @staticmethod
26
+ def get_kv_cache_shape(
27
+ num_blocks: int,
28
+ block_size: int,
29
+ num_kv_heads: int,
30
+ head_size: int,
31
+ ) -> Tuple[int, ...]:
32
+ return PagedAttention.get_kv_cache_shape(num_blocks, block_size,
33
+ num_kv_heads, head_size)
34
+
35
+ @staticmethod
36
+ def swap_blocks(
37
+ src_kv_cache: torch.Tensor,
38
+ dst_kv_cache: torch.Tensor,
39
+ src_to_dst: Dict[int, int],
40
+ ) -> None:
41
+ PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
42
+
43
+ @staticmethod
44
+ def copy_blocks(
45
+ kv_caches: List[torch.Tensor],
46
+ src_to_dists: Dict[int, List[int]],
47
+ ) -> None:
48
+ PagedAttention.copy_blocks(kv_caches, src_to_dists)
49
+
50
+
51
+@dataclass
52
+class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
53
+ """Metadata for TorchSDPABackend.
54
+ """
55
+ # Currently, input sequences can only contain all prompts
56
+ # or all decoding. True if all sequences are prompts.
57
+ is_prompt: bool
58
+ slot_mapping: torch.Tensor
59
+ prompt_lens: Optional[List[int]]
60
+ prompt_lens_tensor: Optional[torch.Tensor]
61
+ num_prompt_tokens: int
62
+ num_generation_tokens: int
63
+
64
+ max_subquery_len: Optional[int] = None
65
+ max_prompt_len: Optional[int] = None
66
+ subquery_start_loc: Optional[torch.Tensor] = None
67
+ seq_start_loc: Optional[torch.Tensor] = None
68
+ use_cuda_graph: bool = False
69
+
70
+ def __post_init__(self):
71
+ # Set during the execution of the first attention op.
72
+ # It is a list because it is needed to set per prompt
73
+ # when alibi slopes is used. It is because of the limitation
74
+ # from xformer API.
75
+ # will not appear in the __repr__ and __init__
76
+ self.attn_bias: Optional[List[torch.Tensor]] = None
77
+
78
+
79
+class TorchSDPABackendImpl(AttentionImpl):
80
+
81
+ def __init__(
82
+ self,
83
+ num_heads: int,
84
+ head_size: int,
85
+ scale: float,
86
+ num_kv_heads: Optional[int] = None,
87
+ alibi_slopes: Optional[List[float]] = None,
88
+ sliding_window: Optional[int] = None,
89
+ ) -> None:
90
+ self.num_heads = num_heads
91
+ self.head_size = head_size
92
+ self.scale = float(scale)
93
+ self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
94
+ self.sliding_window = sliding_window
95
+ if alibi_slopes is not None:
96
+ assert len(alibi_slopes) == num_heads
97
+ alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
98
+ self.alibi_slopes = alibi_slopes
99
+ self.need_mask = (self.alibi_slopes is not None
100
+ or self.sliding_window is not None)
101
+
102
+ assert self.num_heads % self.num_kv_heads == 0
103
+ self.num_queries_per_kv = self.num_heads // self.num_kv_heads
104
+ suppored_head_sizes = PagedAttention.get_supported_head_sizes()
105
+ if head_size not in suppored_head_sizes:
106
+ raise ValueError(
107
+ f"Head size {head_size} is not supported by PagedAttention. "
108
+ f"Supported head sizes are: {suppored_head_sizes}.")
109
+
110
+ def forward(
111
+ self,
112
+ query: torch.Tensor,
113
+ key: torch.Tensor,
114
+ value: torch.Tensor,
115
+ kv_cache: Optional[torch.Tensor],
116
+ attn_metadata: TorchSDPAMetadata,
117
+ ) -> torch.Tensor:
118
+ """Forward pass with torch SDPA and PagedAttention.
119
+
120
+ Args:
121
+ query: shape = [num_tokens, num_heads * head_size]
122
+ key: shape = [num_tokens, num_kv_heads * head_size]
123
+ value: shape = [num_tokens, num_kv_heads * head_size]
124
+ kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
125
+ attn_metadata: Metadata for attention.
126
+ Returns:
127
+ shape = [num_tokens, num_heads * head_size]
128
+ """
129
+ num_tokens, hidden_size = query.shape
130
+ # Reshape the query, key, and value tensors.
131
+ query = query.view(-1, self.num_heads, self.head_size)
132
+ key = key.view(-1, self.num_kv_heads, self.head_size)
133
+ value = value.view(-1, self.num_kv_heads, self.head_size)
134
+
135
+ if kv_cache is not None:
136
+ key_cache, value_cache = PagedAttention.split_kv_cache(
137
+ kv_cache, self.num_kv_heads, self.head_size)
138
+ PagedAttention.write_to_paged_cache(key, value, key_cache,
139
+ value_cache,
140
+ attn_metadata.slot_mapping,
141
+ attn_metadata.kv_cache_dtype)
142
+
143
+ if attn_metadata.is_prompt:
144
+ if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
145
+ if self.num_kv_heads != self.num_heads:
146
+ key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
147
+ value = value.repeat_interleave(self.num_queries_per_kv,
148
+ dim=1)
149
+
150
+ if attn_metadata.attn_bias is None:
151
+ if self.alibi_slopes is not None:
152
+ att_masks = _make_alibi_bias(
153
+ self.alibi_slopes, query.dtype,
154
+ attn_metadata.prompt_lens) # type: ignore
155
+ elif self.sliding_window is not None:
156
+ att_masks = _make_sliding_window_bias(
157
+ attn_metadata.prompt_lens, self.sliding_window,
158
+ query.dtype) # type: ignore
159
+ else:
160
+ att_masks = [None] * len(attn_metadata.prompt_lens)
161
+ attn_metadata.attn_bias = att_masks
162
+
163
+ query = query.movedim(0, query.dim() - 2)
164
+ key = key.movedim(0, key.dim() - 2)
165
+ value = value.movedim(0, value.dim() - 2)
166
+
167
+ start = 0
168
+ output = torch.empty(
169
+ (num_tokens, self.num_heads, self.head_size),
170
+ dtype=query.dtype)
171
+ for prompt_len, mask in zip(attn_metadata.prompt_lens,
172
+ attn_metadata.attn_bias):
173
+ end = start + prompt_len
174
+ sub_out = scaled_dot_product_attention(
175
+ query[:, start:end, :],
176
+ key[:, start:end, :],
177
+ value[:, start:end, :],
178
+ attn_mask=mask,
179
+ dropout_p=0.0,
180
+ is_causal=not self.need_mask,
181
+ scale=self.scale).movedim(query.dim() - 2, 0)
182
+ output[start:end, :, :] = sub_out
183
+ start = end
184
+ else:
185
+ # prefix-enabled attention
186
+ raise RuntimeError(
187
+ "Torch SDPA backend doesn't support prefix decoding.")
188
+
189
+ else:
190
+ # Decoding run.
191
+ output = PagedAttention.forward_decode(
192
+ query,
193
+ key_cache,
194
+ value_cache,
195
+ attn_metadata.block_tables,
196
+ attn_metadata.context_lens,
197
+ attn_metadata.max_context_len,
198
+ attn_metadata.kv_cache_dtype,
199
+ self.num_kv_heads,
200
+ self.scale,
201
+ self.alibi_slopes,
202
+ )
203
+
204
+ # Reshape the output tensor.
205
+ return output.view(-1, self.num_heads * self.head_size)
206
+
207
+
208
+def _make_alibi_bias(
209
+ alibi_slopes: torch.Tensor,
210
+ dtype: torch.dtype,
211
+ prompt_lens: List[int],
212
+) -> List[torch.Tensor]:
213
+ attn_biases = []
214
+ for prompt_len in prompt_lens:
215
+ bias = torch.arange(prompt_len, dtype=dtype)
216
+ # NOTE(zhuohan): HF uses
217
+ # `bias = bias[None, :].repeat(prompt_len, 1)`
218
+ # here. We find that both biases give the same results, but
219
+ # the bias below more accurately follows the original ALiBi
220
+ # paper.
221
+ bias = bias[None, :] - bias[:, None]
222
+
223
+ num_heads = alibi_slopes.shape[0]
224
+ bias = bias[None, :].expand(num_heads, prompt_len, prompt_len)
225
+ bias.mul_(alibi_slopes[:, None, None])
226
+ inf_mask = torch.empty(
227
+ (1, prompt_len, prompt_len),
228
+ dtype=bias.dtype).fill_(-torch.inf).triu_(diagonal=1)
229
+ attn_biases.append((bias + inf_mask).to(dtype))
230
+
231
+ return attn_biases
232
+
233
+
234
+def _make_sliding_window_bias(
235
+ prompt_lens: List[int],
236
+ window_size: Optional[int],
237
+ dtype: torch.dtype,
238
+) -> List[torch.Tensor]:
239
+ attn_biases = []
240
+ for prompt_len in prompt_lens:
241
+ tensor = torch.full(
242
+ (1, prompt_len, prompt_len),
243
+ dtype=dtype,
244
+ fill_value=1,
245
+ )
246
+ shift = 0
247
+ mask = torch.tril(tensor, diagonal=shift).to(dtype) # type: ignore
248
+ if window_size is not None:
249
+ mask = torch.triu(mask, diagonal=shift - window_size + 1)
250
+ mask = torch.log(mask)
251
+ attn_biases.append(mask.to(dtype))
252
+
253
+ return attn_biases
vllm/attention/selector.py
+7
-1
@@ -5,7 +5,7 @@ import torch
5
6
from vllm.attention.backends.abstract import AttentionBackend
7
from vllm.logger import init_logger
8
-from vllm.utils import is_hip
8
+from vllm.utils import is_cpu, is_hip
9
10
logger = init_logger(__name__)
11
@@ -17,6 +17,10 @@ def get_attn_backend(dtype: torch.dtype) -> Type[AttentionBackend]:
17
from vllm.attention.backends.flash_attn import ( # noqa: F401
18
FlashAttentionBackend)
19
return FlashAttentionBackend
20
+ elif is_cpu():
21
+ logger.info("Using Torch SDPA backend.")
22
+ from vllm.attention.backends.torch_sdpa import TorchSDPABackend
23
+ return TorchSDPABackend
24
else:
25
logger.info("Using XFormers backend.")
26
from vllm.attention.backends.xformers import ( # noqa: F401
@@ -29,6 +33,8 @@ def _can_use_flash_attn(dtype: torch.dtype) -> bool:
33
# AMD GPUs.
34
logger.info("Cannot use FlashAttention backend for AMD GPUs.")
35
return False
36
+ if is_cpu():
37
+ return False
38
if torch.cuda.get_device_capability()[0] < 8:
39
# Volta and Turing NVIDIA GPUs.
40
logger.info("Cannot use FlashAttention backend for Volta and Turing "
vllm/config.py
+4
-1
@@ -10,7 +10,8 @@ from transformers import PretrainedConfig
10
11
from vllm.logger import init_logger
12
from vllm.transformers_utils.config import get_config, get_hf_text_config
13
-from vllm.utils import get_cpu_memory, get_nvcc_cuda_version, is_hip, is_neuron
13
+from vllm.utils import (get_cpu_memory, get_nvcc_cuda_version, is_cpu, is_hip,
14
+ is_neuron)
15
16
if TYPE_CHECKING:
17
from ray.util.placement_group import PlacementGroup
@@ -598,6 +599,8 @@ class DeviceConfig:
599
# Automated device type detection
600
if is_neuron():
601
self.device_type = "neuron"
602
+ elif is_cpu():
603
+ self.device_type = "cpu"
604
else:
605
# We don't call torch.cuda.is_available() here to
606
# avoid initializing CUDA before workers are forked
vllm/engine/arg_utils.py
+1
-1
@@ -332,7 +332,7 @@ class EngineArgs:
332
parser.add_argument("--device",
333
type=str,
334
default=EngineArgs.device,
335
- choices=["auto", "cuda", "neuron"],
335
+ choices=["auto", "cuda", "neuron", "cpu"],
336
help='Device type for vLLM execution.')
337
# Related to Vision-language models such as llava
338
parser.add_argument(
vllm/engine/llm_engine.py
+3
@@ -178,6 +178,9 @@ class LLMEngine:
178
if device_config.device_type == "neuron":
179
from vllm.executor.neuron_executor import NeuronExecutor
180
executor_class = NeuronExecutor
181
+ elif device_config.device_type == "cpu":
182
+ from vllm.executor.cpu_executor import CPUExecutor
183
+ executor_class = CPUExecutor
184
elif parallel_config.worker_use_ray:
185
initialize_ray_cluster(parallel_config)
186
from vllm.executor.ray_gpu_executor import RayGPUExecutor
vllm/executor/cpu_executor.py
new
+154
@@ -0,0 +1,154 @@
1
+import os
2
+from typing import Dict, List, Optional
3
+
4
+import torch
5
+
6
+from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
7
+ ParallelConfig, SchedulerConfig)
8
+from vllm.executor.executor_base import ExecutorBase
9
+from vllm.logger import init_logger
10
+from vllm.lora.request import LoRARequest
11
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
12
+from vllm.utils import get_distributed_init_method, get_ip, get_open_port
13
+
14
+logger = init_logger(__name__)
15
+
16
+
17
+class CPUExecutor(ExecutorBase):
18
+
19
+ def __init__(self, model_config: ModelConfig, cache_config: CacheConfig,
20
+ parallel_config: ParallelConfig,
21
+ scheduler_config: SchedulerConfig,
22
+ device_config: DeviceConfig,
23
+ lora_config: Optional[LoRAConfig], *args, **kwargs) -> None:
24
+ assert device_config.device_type == "cpu"
25
+ assert lora_config is None, "cpu backend doesn't support LoRA"
26
+ model_config = _verify_and_get_model_config(model_config)
27
+ cache_config = _verify_and_get_cache_config(cache_config)
28
+
29
+ self.model_config = model_config
30
+ self.cache_config = cache_config
31
+ self.lora_config = lora_config
32
+ self.parallel_config = parallel_config
33
+ self.scheduler_config = scheduler_config
34
+ self.device_config = device_config
35
+
36
+ # Instantiate the worker and load the model to CPU.
37
+ self._init_worker()
38
+ self._init_cache()
39
+
40
+ def _init_worker(self):
41
+ from vllm.worker.cpu_worker import CPUWorker
42
+
43
+ assert self.parallel_config.world_size == 1, (
44
+ "CPUExecutor only supports single CPU socket currently.")
45
+
46
+ distributed_init_method = get_distributed_init_method(
47
+ get_ip(), get_open_port())
48
+ self.driver_worker = CPUWorker(
49
+ self.model_config,
50
+ self.parallel_config,
51
+ self.scheduler_config,
52
+ self.device_config,
53
+ local_rank=0,
54
+ rank=0,
55
+ distributed_init_method=distributed_init_method,
56
+ lora_config=self.lora_config,
57
+ kv_cache_dtype=self.cache_config.cache_dtype,
58
+ is_driver_worker=True,
59
+ )
60
+ self.driver_worker.init_device()
61
+ self.driver_worker.load_model()
62
+
63
+ def _init_cache(self) -> None:
64
+ num_cpu_blocks = self.driver_worker.get_cpu_cache_block_num(
65
+ block_size=self.cache_config.block_size,
66
+ cache_space=self.cache_config.cpu_kvcache_space_bytes,
67
+ cache_dtype=self.cache_config.cache_dtype,
68
+ )
69
+
70
+ logger.info(f"# CPU blocks: {num_cpu_blocks}")
71
+ if num_cpu_blocks <= 0:
72
+ raise ValueError("No available memory for the cache blocks. "
73
+ "Try increasing `VLLM_CPU_KVCACHE_SPACE` when "
74
+ "initializing the engine.")
75
+
76
+ max_seq_len = self.cache_config.block_size * num_cpu_blocks
77
+ if self.model_config.max_model_len > max_seq_len:
78
+ raise ValueError(
79
+ f"The model's max seq len ({self.model_config.max_model_len}) "
80
+ "is larger than the maximum number of tokens that can be "
81
+ f"stored in KV cache ({max_seq_len}). Try increasing "
82
+ "`VLLM_CPU_KVCACHE_SPACE` or decreasing `max_model_len` when "
83
+ "initializing the engine.")
84
+
85
+ # Note: To reuse the cache management procedure,
86
+ # use cpu cache as 'gpu cache'.
87
+ self.cache_config.num_gpu_blocks = num_cpu_blocks # type: ignore
88
+ self.cache_config.num_cpu_blocks = 0 # type: ignore
89
+
90
+ # Initialize the cache.
91
+ self.driver_worker.init_cache_engine(cache_config=self.cache_config)
92
+
93
+ def execute_model(self,
94
+ seq_group_metadata_list: List[SequenceGroupMetadata],
95
+ blocks_to_swap_in: Dict[int, int],
96
+ blocks_to_swap_out: Dict[int, int],
97
+ blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
98
+ output = self.driver_worker.execute_model(
99
+ seq_group_metadata_list=seq_group_metadata_list,
100
+ blocks_to_swap_in=blocks_to_swap_in,
101
+ blocks_to_swap_out=blocks_to_swap_out,
102
+ blocks_to_copy=blocks_to_copy,
103
+ )
104
+ return output
105
+
106
+ def add_lora(self, lora_request: LoRARequest) -> bool:
107
+ raise NotImplementedError("LoRA is not implemented for cpu backend.")
108
+
109
+ def remove_lora(self, lora_id: int) -> bool:
110
+ raise NotImplementedError("LoRA is not implemented for cpu backend.")
111
+
112
+ def list_loras(self) -> List[int]:
113
+ raise NotImplementedError("LoRA is not implemented for cpu backend.")
114
+
115
+ def check_health(self) -> None:
116
+ # CPUExecutor will always be healthy as long as
117
+ # it's running.
118
+ return
119
+
120
+
121
+def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
122
+ if config.dtype == torch.float16:
123
+ logger.warning("float16 is not supported on CPU, casting to bfloat16.")
124
+ config.dtype = torch.bfloat16
125
+ if not config.enforce_eager:
126
+ logger.warning(
127
+ "CUDA graph is not supported on CPU, fallback to the eager "
128
+ "mode.")
129
+ config.enforce_eager = True
130
+ return config
131
+
132
+
133
+def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
134
+ _GB = 1 << 30
135
+ if config.enable_prefix_caching:
136
+ logger.warning("Prefix caching is not supported on CPU, disable it.")
137
+ config.enable_prefix_caching = False
138
+
139
+ kv_cache_space_str = os.getenv("VLLM_CPU_KVCACHE_SPACE", "0")
140
+ kv_cache_space = int(kv_cache_space_str)
141
+
142
+ if kv_cache_space >= 0:
143
+ if kv_cache_space == 0:
144
+ config.cpu_kvcache_space_bytes = 4 * _GB # type: ignore
145
+ logger.warning("Environment variable VLLM_CPU_KVCACHE_SPACE (GB) "
146
+ "for CPU backend is not set, using 4 by default.")
147
+ else:
148
+ config.cpu_kvcache_space_bytes = kv_cache_space * _GB # type: ignore
149
+ else:
150
+ raise RuntimeError(
151
+ "Invalid environment variable VLLM_CPU_KVCACHE_SPACE"
152
+ f" {kv_cache_space}, expect a positive integer value.")
153
+
154
+ return config
vllm/utils.py
+10
@@ -117,6 +117,13 @@ def is_hip() -> bool:
117
return torch.version.hip is not None
118
119
120
+@lru_cache(maxsize=None)
121
+def is_cpu() -> bool:
122
+ from importlib.metadata import version
123
+ is_cpu_flag = "cpu" in version("vllm")
124
+ return is_cpu_flag
125
+
126
+
127
@lru_cache(maxsize=None)
128
def is_neuron() -> bool:
129
try:
@@ -362,6 +369,9 @@ def is_pin_memory_available() -> bool:
369
elif is_neuron():
370
print_warning_once("Pin memory is not supported on Neuron.")
371
return False
372
+ elif is_cpu():
373
+ print_warning_once("Pin memory is not supported on CPU.")
374
+ return False
375
return True
376
377
vllm/worker/cpu_worker.py
new
+280
@@ -0,0 +1,280 @@
1
+"""A CPU worker class."""
2
+from typing import Dict, List, Optional
3
+
4
+import torch
5
+import torch.distributed
6
+
7
+from vllm.attention import get_attn_backend
8
+from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9
+ ParallelConfig, SchedulerConfig)
10
+from vllm.logger import init_logger
11
+from vllm.model_executor import set_random_seed
12
+from vllm.model_executor.model_loader import get_model
13
+from vllm.model_executor.parallel_utils.communication_op import (
14
+ broadcast_tensor_dict)
15
+from vllm.model_executor.parallel_utils.parallel_state import (
16
+ ensure_model_parallel_initialized)
17
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
18
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
19
+from vllm.worker.model_runner import ModelRunner
20
+
21
+logger = init_logger(__name__)
22
+
23
+
24
+class CPUModelRunner(ModelRunner):
25
+
26
+ def load_model(self) -> None:
27
+ self.model = get_model(self.model_config,
28
+ self.device_config,
29
+ lora_config=self.lora_config,
30
+ parallel_config=self.parallel_config,
31
+ scheduler_config=self.scheduler_config)
32
+
33
+
34
+class CPUCacheEngine:
35
+ """Manages the KV cache for CPU backend.
36
+
37
+ This class is responsible for initializing and managing CPU KV
38
+ caches. It also provides methods for performing KV cache operations, such
39
+ as copying.
40
+ """
41
+
42
+ def __init__(self, cache_config: CacheConfig, model_config: ModelConfig,
43
+ parallel_config: ParallelConfig,
44
+ device_config: DeviceConfig) -> None:
45
+ assert device_config.device_type == "cpu"
46
+ self.cache_config = cache_config
47
+ self.model_config = model_config
48
+ self.parallel_config = parallel_config
49
+
50
+ self.head_size = model_config.get_head_size()
51
+ self.num_layers = model_config.get_num_layers(parallel_config)
52
+ self.num_heads = model_config.get_num_kv_heads(parallel_config)
53
+
54
+ self.block_size = cache_config.block_size
55
+ # Note: In CacheConfig, num_gpu_blocks actual is num_cpu_blocks
56
+ # for CPU backend, because we want to reuse KV cache management
57
+ # in the scheduler.
58
+ self.num_cpu_blocks = cache_config.num_gpu_blocks
59
+
60
+ if cache_config.cache_dtype == "auto":
61
+ self.dtype = model_config.dtype
62
+ else:
63
+ self.dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
64
+
65
+ # Get attention backend.
66
+ self.attn_backend = get_attn_backend(model_config.dtype)
67
+
68
+ # Initialize the cache.
69
+ self.cpu_cache = self._allocate_kv_cache(self.num_cpu_blocks)
70
+
71
+ def _allocate_kv_cache(
72
+ self,
73
+ num_blocks: int,
74
+ ) -> List[torch.Tensor]:
75
+ """Allocates KV cache on CPU."""
76
+ kv_cache_shape = self.attn_backend.get_kv_cache_shape(
77
+ num_blocks, self.block_size, self.num_heads, self.head_size)
78
+ kv_cache: List[torch.Tensor] = []
79
+ for _ in range(self.num_layers):
80
+ kv_cache.append(
81
+ torch.empty(kv_cache_shape, dtype=self.dtype, device="cpu"))
82
+ return kv_cache
83
+
84
+ def swap_in(self, src_to_dst: Dict[int, int]) -> None:
85
+ raise NotImplementedError("Swap is not supported in CPUCacheEngine.")
86
+
87
+ def swap_out(self, src_to_dst: Dict[int, int]) -> None:
88
+ raise NotImplementedError("Swap is not supported in CPUCacheEngine.")
89
+
90
+ def copy(self, src_to_dsts: Dict[int, List[int]]) -> None:
91
+ self.attn_backend.copy_blocks(self.cpu_cache, src_to_dsts)
92
+
93
+ @staticmethod
94
+ def get_cache_block_size(
95
+ block_size: int,
96
+ cache_dtype: str,
97
+ model_config: ModelConfig,
98
+ parallel_config: ParallelConfig,
99
+ ) -> int:
100
+ head_size = model_config.get_head_size()
101
+ num_heads = model_config.get_num_kv_heads(parallel_config)
102
+ num_layers = model_config.get_num_layers(parallel_config)
103
+
104
+ key_cache_block = block_size * num_heads * head_size
105
+ value_cache_block = key_cache_block
106
+ total = num_layers * (key_cache_block + value_cache_block)
107
+ if cache_dtype == "auto":
108
+ dtype = model_config.dtype
109
+ else:
110
+ dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype]
111
+ dtype_size = torch.tensor([], dtype=dtype).element_size()
112
+ return dtype_size * total
113
+
114
+
115
+class CPUWorker:
116
+ """A worker class that executes (a partition of) the model on a CPU socket.
117
+
118
+ Each worker is associated with a single CPU socket. The worker is
119
+ responsible for maintaining the KV cache and executing the model on the
120
+ CPU. In case of distributed inference, each worker is assigned a partition
121
+ of the model.
122
+ """
123
+
124
+ def __init__(
125
+ self,
126
+ model_config: ModelConfig,
127
+ parallel_config: ParallelConfig,
128
+ scheduler_config: SchedulerConfig,
129
+ device_config: DeviceConfig,
130
+ local_rank: int,
131
+ rank: int,
132
+ distributed_init_method: str,
133
+ lora_config: Optional[LoRAConfig] = None,
134
+ kv_cache_dtype: Optional[str] = "auto",
135
+ is_driver_worker: bool = False,
136
+ ) -> None:
137
+ self.model_config = model_config
138
+ self.parallel_config = parallel_config
139
+ self.scheduler_config = scheduler_config
140
+ self.device_config = device_config
141
+ self.local_rank = local_rank
142
+ self.rank = rank
143
+ self.distributed_init_method = distributed_init_method
144
+ self.lora_config = lora_config
145
+ self.is_driver_worker = is_driver_worker
146
+ if self.is_driver_worker:
147
+ assert self.rank == 0, "The driver worker must have rank 0."
148
+
149
+ self.model_runner = CPUModelRunner(model_config,
150
+ parallel_config,
151
+ scheduler_config,
152
+ device_config,
153
+ lora_config=self.lora_config,
154
+ kv_cache_dtype=kv_cache_dtype,
155
+ is_driver_worker=is_driver_worker)
156
+ # Uninitialized cache engine. Will be initialized by
157
+ # self.init_cache_engine().
158
+ self.cache_config = None
159
+ self.cache_engine = None
160
+ self.cpu_cache = None
161
+
162
+ def init_device(self) -> None:
163
+ self.init_distributed_environment()
164
+ # Set random seed.
165
+ set_random_seed(self.model_config.seed)
166
+
167
+ def load_model(self):
168
+ self.model_runner.load_model()
169
+
170
+ def get_cpu_cache_block_num(
171
+ self,
172
+ block_size: int,
173
+ cache_space: int,
174
+ cache_dtype: str,
175
+ ) -> int:
176
+ """
177
+ Args:
178
+ block_size: The size of the cache block.
179
+ cache_space: The size of the CPU KV cache space in bytes.
180
+ """
181
+ # For CPU device, the block number will be calculated based on the
182
+ # cpu_kvcache_space.
183
+ cache_block_size = CPUCacheEngine.get_cache_block_size(
184
+ block_size, cache_dtype, self.model_config, self.parallel_config)
185
+ num_cpu_blocks = int(cache_space // cache_block_size)
186
+ num_cpu_blocks = max(num_cpu_blocks, 0)
187
+
188
+ return num_cpu_blocks
189
+
190
+ def init_cache_engine(self, cache_config: CacheConfig) -> None:
191
+ self.cache_config = cache_config
192
+ self.cache_engine = CPUCacheEngine(self.cache_config,
193
+ self.model_config,
194
+ self.parallel_config,
195
+ self.device_config)
196
+ self.cpu_cache = self.cache_engine.cpu_cache
197
+ self.model_runner.block_size = self.cache_engine.block_size
198
+
199
+ assert self.cpu_cache is not None
200
+
201
+ # Populate the cache to warmup the memory
202
+ for layer_cache in self.cpu_cache:
203
+ layer_cache.fill_(0)
204
+
205
+ def cache_copy(
206
+ self,
207
+ blocks_to_copy: Dict[int, List[int]],
208
+ ) -> None:
209
+ if blocks_to_copy:
210
+ self.cache_engine.copy(blocks_to_copy)
211
+
212
+ @torch.inference_mode()
213
+ def execute_model(
214
+ self,
215
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None,
216
+ blocks_to_swap_in: Optional[Dict[int, int]] = None,
217
+ blocks_to_swap_out: Optional[Dict[int, int]] = None,
218
+ blocks_to_copy: Optional[Dict[int, List[int]]] = None,
219
+ ) -> Optional[SamplerOutput]:
220
+ if self.is_driver_worker:
221
+ assert seq_group_metadata_list is not None
222
+ num_seq_groups = len(seq_group_metadata_list)
223
+ assert blocks_to_swap_in is not None
224
+ assert blocks_to_swap_out is not None
225
+ assert blocks_to_copy is not None
226
+ assert len(blocks_to_swap_in) == 0
227
+ assert len(blocks_to_swap_out) == 0
228
+ data = {
229
+ "num_seq_groups": num_seq_groups,
230
+ "blocks_to_copy": blocks_to_copy,
231
+ }
232
+ broadcast_tensor_dict(data, src=0)
233
+ else:
234
+ data = broadcast_tensor_dict(src=0)
235
+ num_seq_groups = data["num_seq_groups"]
236
+ blocks_to_copy = data["blocks_to_copy"]
237
+
238
+ self.cache_copy(blocks_to_copy)
239
+
240
+ # If there is no input, we don't need to execute the model.
241
+ if num_seq_groups == 0:
242
+ return {}
243
+
244
+ output = self.model_runner.execute_model(seq_group_metadata_list,
245
+ self.cpu_cache)
246
+ return output
247
+
248
+ def init_distributed_environment(self) -> None:
249
+ """Initialize the distributed environment."""
250
+
251
+ parallel_config = self.parallel_config
252
+ rank = self.rank
253
+ distributed_init_method = self.distributed_init_method
254
+
255
+ if torch.distributed.is_initialized():
256
+ torch_world_size = torch.distributed.get_world_size()
257
+ if torch_world_size != parallel_config.world_size:
258
+ raise RuntimeError(
259
+ "torch.distributed is already initialized but the torch "
260
+ "world size does not match parallel_config.world_size "
261
+ f"({torch_world_size} vs. {parallel_config.world_size}).")
262
+ elif not distributed_init_method:
263
+ raise ValueError(
264
+ "distributed_init_method must be set if torch.distributed "
265
+ "is not already initialized")
266
+ else:
267
+ backend = "gloo"
268
+ torch.distributed.init_process_group(
269
+ backend=backend,
270
+ world_size=parallel_config.world_size,
271
+ rank=rank,
272
+ init_method=distributed_init_method,
273
+ )
274
+
275
+ # A small all_reduce for warmup.
276
+ torch.distributed.all_reduce(torch.zeros(1).cpu())
277
+
278
+ ensure_model_parallel_initialized(
279
+ parallel_config.tensor_parallel_size,
280
+ parallel_config.pipeline_parallel_size)