A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Feature] vLLM ARM Enablement for AARCH64 CPUs (#9228)
Signed-off-by: Sanket Kale <sanketk.kale@fujitsu.com> Co-authored-by: Sanket Kale <sanketk.kale@fujitsu.com> Co-authored-by: mgoin <michael@neuralmagic.com>
Sanket Kale committed
Nov 26, 2024 at 08:02 UTC
a6760f6456b714409685e23301c820a85da856ca
9 files changed
+678
-16
Dockerfile.arm
new
+62
@@ -0,0 +1,62 @@
1
+# This vLLM Dockerfile is used to construct an image that can build and run vLLM on ARM CPU platform.
2
+
3
+FROM ubuntu:22.04 AS cpu-test-arm
4
+
5
+ENV CCACHE_DIR=/root/.cache/ccache
6
+
7
+ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache
8
+
9
+RUN --mount=type=cache,target=/var/cache/apt \
10
+ apt-get update -y \
11
+ && apt-get install -y curl ccache git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 libnuma-dev \
12
+ && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \
13
+ && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
14
+
15
+# tcmalloc provides better memory allocation efficiency, e.g., holding memory in caches to speed up access of commonly-used objects.
16
+RUN --mount=type=cache,target=/root/.cache/pip \
17
+ pip install py-cpuinfo # Use this to gather CPU info and optimize based on ARM Neoverse cores
18
+
19
+# Set LD_PRELOAD for tcmalloc on ARM
20
+ENV LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libtcmalloc_minimal.so.4"
21
+
22
+RUN echo 'ulimit -c 0' >> ~/.bashrc
23
+
24
+WORKDIR /workspace
25
+
26
+ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu"
27
+ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
28
+RUN --mount=type=cache,target=/root/.cache/pip \
29
+ --mount=type=bind,src=requirements-build.txt,target=requirements-build.txt \
30
+ pip install --upgrade pip && \
31
+ pip install -r requirements-build.txt
32
+
33
+FROM cpu-test-arm AS build
34
+
35
+WORKDIR /workspace/vllm
36
+
37
+RUN --mount=type=cache,target=/root/.cache/pip \
38
+ --mount=type=bind,src=requirements-common.txt,target=requirements-common.txt \
39
+ --mount=type=bind,src=requirements-cpu.txt,target=requirements-cpu.txt \
40
+ pip install -v -r requirements-cpu.txt
41
+
42
+COPY . .
43
+ARG GIT_REPO_CHECK=0
44
+RUN --mount=type=bind,source=.git,target=.git \
45
+ if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
46
+
47
+# Disabling AVX512 specific optimizations for ARM
48
+ARG VLLM_CPU_DISABLE_AVX512="true"
49
+ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512}
50
+
51
+RUN --mount=type=cache,target=/root/.cache/pip \
52
+ --mount=type=cache,target=/root/.cache/ccache \
53
+ --mount=type=bind,source=.git,target=.git \
54
+ VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel && \
55
+ pip install dist/*.whl && \
56
+ rm -rf dist
57
+
58
+WORKDIR /workspace/
59
+
60
+RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
61
+
62
+ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
\ No newline at end of file
cmake/cpu_extension.cmake
+24
-9
@@ -16,16 +16,15 @@ include_directories("${CMAKE_SOURCE_DIR}/csrc")
16
#
17
# Check the compile flags
18
#
19
-if (CMAKE_SYSTEM_PROCESSOR STREQUAL "ppc64le")
20
- list(APPEND CXX_COMPILE_FLAGS
21
- "-fopenmp"
22
- "-DVLLM_CPU_EXTENSION")
23
-else()
19
+
20
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
21
list(APPEND CXX_COMPILE_FLAGS
25
- "-fopenmp"
22
"-mf16c"
27
- "-DVLLM_CPU_EXTENSION")
23
+ )
24
endif()
25
+list(APPEND CXX_COMPILE_FLAGS
26
+ "-fopenmp"
27
+ "-DVLLM_CPU_EXTENSION")
28
29
execute_process(COMMAND cat /proc/cpuinfo
30
RESULT_VARIABLE CPUINFO_RET
@@ -59,6 +58,8 @@ find_isa(${CPUINFO} "avx2" AVX2_FOUND)
58
find_isa(${CPUINFO} "avx512f" AVX512_FOUND)
59
find_isa(${CPUINFO} "POWER10" POWER10_FOUND)
60
find_isa(${CPUINFO} "POWER9" POWER9_FOUND)
61
+find_isa(${CPUINFO} "asimd" ASIMD_FOUND) # Check for ARM NEON support
62
+find_isa(${CPUINFO} "bf16" ARM_BF16_FOUND) # Check for ARM BF16 support
63
64
if (AVX512_FOUND AND NOT AVX512_DISABLED)
65
list(APPEND CXX_COMPILE_FLAGS
@@ -78,9 +79,11 @@ if (AVX512_FOUND AND NOT AVX512_DISABLED)
79
else()
80
message(WARNING "Disable AVX512-BF16 ISA support, no avx512_bf16 found in local CPU flags." " If cross-compilation is required, please set env VLLM_CPU_AVX512BF16=1.")
81
endif()
82
+
83
elseif (AVX2_FOUND)
84
list(APPEND CXX_COMPILE_FLAGS "-mavx2")
85
message(WARNING "vLLM CPU backend using AVX2 ISA")
86
+
87
elseif (POWER9_FOUND OR POWER10_FOUND)
88
message(STATUS "PowerPC detected")
89
# Check for PowerPC VSX support
@@ -88,8 +91,20 @@ elseif (POWER9_FOUND OR POWER10_FOUND)
91
"-mvsx"
92
"-mcpu=native"
93
"-mtune=native")
94
+
95
+elseif (ASIMD_FOUND)
96
+ message(STATUS "ARMv8 or later architecture detected")
97
+ if(ARM_BF16_FOUND)
98
+ message(STATUS "BF16 extension detected")
99
+ set(MARCH_FLAGS "-march=armv8.2-a+bf16+dotprod+fp16")
100
+ add_compile_definitions(ARM_BF16_SUPPORT)
101
+ else()
102
+ message(WARNING "BF16 functionality is not available")
103
+ set(MARCH_FLAGS "-march=armv8.2-a+dotprod+fp16")
104
+ endif()
105
+ list(APPEND CXX_COMPILE_FLAGS ${MARCH_FLAGS})
106
else()
92
- message(FATAL_ERROR "vLLM CPU backend requires AVX512 or AVX2 or Power9+ ISA support.")
107
+ message(FATAL_ERROR "vLLM CPU backend requires AVX512, AVX2, Power9+ ISA or ARMv8 support.")
108
endif()
109
110
#
@@ -159,4 +174,4 @@ define_gpu_extension_target(
174
WITH_SOABI
175
)
176
162
-message(STATUS "Enabling C extension.")
177
+message(STATUS "Enabling C extension.")
\ No newline at end of file
csrc/cpu/attention.cpp
+17
-1
@@ -51,6 +51,10 @@ struct KernelVecType<c10::BFloat16> {
51
using v_load_vec_type = vec_op::BF16Vec16;
52
};
53
#else
54
+ #ifdef __aarch64__
55
+ #ifndef ARM_BF16_SUPPORT
56
+ // pass
57
+ #else
58
template <>
59
struct KernelVecType<c10::BFloat16> {
60
using q_load_vec_type = vec_op::BF16Vec8;
@@ -60,6 +64,18 @@ struct KernelVecType<c10::BFloat16> {
64
using qk_acc_vec_type = vec_op::FP32Vec16;
65
using v_load_vec_type = vec_op::BF16Vec16;
66
};
67
+ #endif
68
+ #else
69
+template <>
70
+struct KernelVecType<c10::BFloat16> {
71
+ using q_load_vec_type = vec_op::BF16Vec8;
72
+ using q_vec_type = vec_op::FP32Vec16;
73
+ using k_load_vec_type = vec_op::BF16Vec16;
74
+ using k_vec_type = vec_op::FP32Vec16;
75
+ using qk_acc_vec_type = vec_op::FP32Vec16;
76
+ using v_load_vec_type = vec_op::BF16Vec16;
77
+};
78
+ #endif
79
#endif
80
81
template <typename T>
@@ -779,4 +795,4 @@ void paged_attention_v2(
795
CALL_V2_KERNEL_LAUNCHER_BLOCK_SIZE(scalar_t);
796
CPU_KERNEL_GUARD_OUT(paged_attention_v2_impl)
797
});
782
-}
798
+}
\ No newline at end of file
csrc/cpu/cpu_types.hpp
+4
-2
@@ -1,4 +1,3 @@
1
-
1
#ifndef CPU_TYPES_HPP
2
#define CPU_TYPES_HPP
3
@@ -8,8 +7,11 @@
7
#elif defined(__POWER9_VECTOR__)
8
//ppc implementation
9
#include "cpu_types_vsx.hpp"
10
+#elif defined(__aarch64__)
11
+ //arm implementation
12
+ #include "cpu_types_arm.hpp"
13
#else
14
#warning "unsupported vLLM cpu implementation"
15
#endif
16
15
-#endif
17
+#endif
\ No newline at end of file
csrc/cpu/cpu_types_arm.hpp
new
+515
@@ -0,0 +1,515 @@
1
+#include <arm_neon.h>
2
+#include <torch/all.h>
3
+#include <cmath>
4
+
5
+namespace vec_op {
6
+
7
+#ifdef ARM_BF16_SUPPORT
8
+ #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
9
+ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
10
+ AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
11
+ AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
12
+#else
13
+ #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
14
+ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
15
+ AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__)
16
+#endif
17
+
18
+#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
19
+ AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
20
+
21
+#ifndef CPU_OP_GUARD
22
+#define CPU_KERNEL_GUARD_IN(NAME)
23
+#define CPU_KERNEL_GUARD_OUT(NAME)
24
+#else
25
+#define CPU_KERNEL_GUARD_IN(NAME) \
26
+ std::cout << #NAME << " invoked." << std::endl;
27
+#define CPU_KERNEL_GUARD_OUT(NAME) std::cout << #NAME << " exit." << std::endl;
28
+#endif
29
+
30
+#define FORCE_INLINE __attribute__((always_inline)) inline
31
+
32
+namespace {
33
+ template <typename T, T... indexes, typename F>
34
+ constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F &&f) {
35
+ (f(std::integral_constant<T, indexes>{}), ...);
36
+ };
37
+};
38
+
39
+template <typename T, T count, typename F,
40
+ typename = std::enable_if_t<std::is_invocable_v<F, T>>>
41
+constexpr void unroll_loop(F &&f) {
42
+ unroll_loop_item(std::make_integer_sequence<T, count>{}, std::forward<F>(f));
43
+}
44
+
45
+template <typename T> struct Vec {
46
+ constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; };
47
+};
48
+
49
+struct FP32Vec8;
50
+struct FP32Vec16;
51
+
52
+struct FP16Vec8 : public Vec<FP16Vec8> {
53
+ constexpr static int VEC_ELEM_NUM = 8;
54
+
55
+ float16x8_t reg;
56
+
57
+ explicit FP16Vec8(const void *ptr)
58
+ : reg(vld1q_f16(static_cast<const __fp16 *>(ptr))) {};
59
+
60
+ explicit FP16Vec8(const FP32Vec8 &);
61
+
62
+ void save(void *ptr) const {
63
+ vst1q_f16(static_cast<__fp16 *>(ptr), reg);
64
+ }
65
+};
66
+
67
+struct FP16Vec16 : public Vec<FP16Vec16> {
68
+ constexpr static int VEC_ELEM_NUM = 16;
69
+
70
+ float16x8x2_t reg;
71
+
72
+ explicit FP16Vec16(const void *ptr) {
73
+ reg.val[0] = vld1q_f16(reinterpret_cast<const __fp16*>(ptr));
74
+ reg.val[1] = vld1q_f16(reinterpret_cast<const __fp16*>(ptr) + 8);
75
+ }
76
+
77
+ explicit FP16Vec16(const FP32Vec16& vec);
78
+
79
+ void save(void *ptr) const {
80
+ vst1q_f16(reinterpret_cast<__fp16*>(ptr), reg.val[0]);
81
+ vst1q_f16(reinterpret_cast<__fp16*>(ptr) + 8, reg.val[1]);
82
+ }
83
+
84
+ void save(void *ptr, const int elem_num) const {
85
+ int full_blocks = elem_num / 8;
86
+ int remainder = elem_num % 8;
87
+
88
+ if (full_blocks > 0) {
89
+ vst1q_f16(reinterpret_cast<__fp16*>(ptr), reg.val[0]);
90
+ if (full_blocks > 1) {
91
+ vst1q_f16(reinterpret_cast<__fp16*>(ptr) + 8, reg.val[1]);
92
+ }
93
+ }
94
+
95
+ if (remainder > 0) {
96
+ float16x8_t temp = reg.val[full_blocks];
97
+ for (int i = 0; i < remainder; ++i) {
98
+ reinterpret_cast<__fp16*>(ptr)[full_blocks * 8 + i] = vgetq_lane_f16(temp, i);
99
+ }
100
+ }
101
+ }
102
+};
103
+
104
+
105
+#ifdef ARM_BF16_SUPPORT
106
+struct BF16Vec8 : public Vec<BF16Vec8> {
107
+ constexpr static int VEC_ELEM_NUM = 8;
108
+
109
+ bfloat16x8_t reg;
110
+
111
+ explicit BF16Vec8(const void *ptr)
112
+ : reg(*reinterpret_cast<const bfloat16x8_t *>(ptr)) {};
113
+
114
+ explicit BF16Vec8(bfloat16x8_t data) : reg(data) {};
115
+
116
+ explicit BF16Vec8(const FP32Vec8 &);
117
+
118
+ explicit BF16Vec8(float32x4x2_t v) : reg(vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.val[0]), v.val[1])) {};
119
+
120
+ void save(void *ptr) const { *reinterpret_cast<bfloat16x8_t *>(ptr) = reg; }
121
+};
122
+
123
+struct BF16Vec16 : public Vec<BF16Vec16> {
124
+ constexpr static int VEC_ELEM_NUM = 16;
125
+
126
+ bfloat16x8x2_t reg;
127
+
128
+ explicit BF16Vec16(const void *ptr)
129
+ : reg(*reinterpret_cast<const bfloat16x8x2_t *>(ptr)) {};
130
+
131
+ explicit BF16Vec16(bfloat16x8x2_t data) : reg(data) {};
132
+
133
+ explicit BF16Vec16(const FP32Vec16 &);
134
+
135
+ explicit BF16Vec16(float32x4x4_t v) : reg({
136
+ vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.val[0]), v.val[1]),
137
+ vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.val[2]), v.val[3])
138
+ }){};
139
+
140
+ void save(void *ptr) const { *reinterpret_cast<bfloat16x8x2_t *>(ptr) = reg; };
141
+};
142
+
143
+struct BF16Vec32 : public Vec<BF16Vec32> {
144
+ constexpr static int VEC_ELEM_NUM = 32;
145
+
146
+ bfloat16x8x4_t reg;
147
+
148
+ explicit BF16Vec32(const void *ptr)
149
+ : reg(*reinterpret_cast<const bfloat16x8x4_t *>(ptr)) {};
150
+
151
+ explicit BF16Vec32(bfloat16x8x4_t data) : reg(data) {};
152
+
153
+ explicit BF16Vec32(const BF16Vec8 &vec8_data) : reg({
154
+ vec8_data.reg,
155
+ vec8_data.reg,
156
+ vec8_data.reg,
157
+ vec8_data.reg
158
+ }) {};
159
+
160
+ void save(void *ptr) const { *reinterpret_cast<bfloat16x8x4_t *>(ptr) = reg; };
161
+};
162
+#endif
163
+
164
+struct FP32Vec4 : public Vec<FP32Vec4> {
165
+ constexpr static int VEC_ELEM_NUM = 4;
166
+
167
+ union AliasReg {
168
+ float32x4_t reg;
169
+ float values[VEC_ELEM_NUM];
170
+ };
171
+
172
+ float32x4_t reg;
173
+
174
+ explicit FP32Vec4(float v) : reg(vdupq_n_f32(v)) {};
175
+
176
+ explicit FP32Vec4() : reg(vdupq_n_f32(0.0f)) {};
177
+
178
+ explicit FP32Vec4(const float *ptr) : reg(vld1q_f32(ptr)) {};
179
+
180
+ explicit FP32Vec4(float32x4_t data) : reg(data) {};
181
+
182
+ explicit FP32Vec4(const FP32Vec4 &data) : reg(data.reg) {};
183
+};
184
+
185
+struct FP32Vec8 : public Vec<FP32Vec8> {
186
+ constexpr static int VEC_ELEM_NUM = 8;
187
+ union AliasReg {
188
+ float32x4x2_t reg;
189
+ float values[VEC_ELEM_NUM];
190
+ };
191
+
192
+ float32x4x2_t reg;
193
+
194
+ explicit FP32Vec8(float v) : reg({vmovq_n_f32(v), vmovq_n_f32(v)}) {};
195
+
196
+ explicit FP32Vec8() : reg({vmovq_n_f32(0.0), vmovq_n_f32(0.0)}) {};
197
+
198
+ explicit FP32Vec8(const float *ptr) : reg({vld1q_f32(ptr), vld1q_f32(ptr + 4)}) {};
199
+
200
+ explicit FP32Vec8(float32x4x2_t data) : reg(data) {};
201
+
202
+ explicit FP32Vec8(const FP32Vec8 &data) : reg(data.reg) {};
203
+
204
+ explicit FP32Vec8(const FP16Vec8 &v) {
205
+ reg.val[0] = vcvt_f32_f16(vget_low_f16(v.reg));
206
+ reg.val[1] = vcvt_f32_f16(vget_high_f16(v.reg));
207
+ };
208
+
209
+ explicit FP32Vec8(float16x8_t v) : reg({vcvt_f32_f16(vget_low_f16(v)), vcvt_f32_f16(vget_high_f16(v))}) {};
210
+
211
+ #ifdef ARM_BF16_SUPPORT
212
+
213
+ explicit FP32Vec8(bfloat16x8_t v) : reg({vcvtq_low_f32_bf16(v), vcvtq_high_f32_bf16(v)}) {};
214
+
215
+ explicit FP32Vec8(const BF16Vec8 &v) : reg({vcvtq_low_f32_bf16(v.reg), vcvtq_high_f32_bf16(v.reg)}) {};
216
+
217
+ #endif
218
+
219
+ float reduce_sum() const {
220
+ AliasReg ar;
221
+ ar.reg = reg;
222
+ float answer = 0;
223
+ unroll_loop<int, VEC_ELEM_NUM>([&answer, &ar](int i) { answer += ar.values[i]; });
224
+
225
+ return answer;
226
+ }
227
+
228
+ FP32Vec8 exp() const {
229
+ AliasReg ar;
230
+ ar.reg = reg;
231
+
232
+ float32x2_t exp_vec0 = {expf(ar.values[0]), expf(ar.values[1])};
233
+ float32x2_t exp_vec1 = {expf(ar.values[2]), expf(ar.values[3])};
234
+ float32x2_t exp_vec2 = {expf(ar.values[4]), expf(ar.values[5])};
235
+ float32x2_t exp_vec3 = {expf(ar.values[6]), expf(ar.values[7])};
236
+
237
+ float32x4_t result0 = vcombine_f32(exp_vec0, exp_vec1);
238
+ float32x4_t result1 = vcombine_f32(exp_vec2, exp_vec3);
239
+
240
+ float32x4x2_t result;
241
+ result.val[0] = result0;
242
+ result.val[1] = result1;
243
+
244
+ return FP32Vec8(result);
245
+ }
246
+
247
+ FP32Vec8 tanh() const {
248
+ AliasReg ar;
249
+ ar.reg = reg;
250
+
251
+ float32x2_t tanh_vec0 = {tanhf(ar.values[0]), tanhf(ar.values[1])};
252
+ float32x2_t tanh_vec1 = {tanhf(ar.values[2]), tanhf(ar.values[3])};
253
+ float32x2_t tanh_vec2 = {tanhf(ar.values[4]), tanhf(ar.values[5])};
254
+ float32x2_t tanh_vec3 = {tanhf(ar.values[6]), tanhf(ar.values[7])};
255
+
256
+ float32x4_t result0 = vcombine_f32(tanh_vec0, tanh_vec1);
257
+ float32x4_t result1 = vcombine_f32(tanh_vec2, tanh_vec3);
258
+
259
+ float32x4x2_t result;
260
+ result.val[0] = result0;
261
+ result.val[1] = result1;
262
+
263
+ return FP32Vec8(result);
264
+ }
265
+
266
+ FP32Vec8 er() const {
267
+ AliasReg ar;
268
+ ar.reg = reg;
269
+
270
+ float32x2_t er_vec0 = {static_cast<float32_t>(erf(ar.values[0])), static_cast<float32_t>(erf(ar.values[1]))};
271
+ float32x2_t er_vec1 = {static_cast<float32_t>(erf(ar.values[2])), static_cast<float32_t>(erf(ar.values[3]))};
272
+ float32x2_t er_vec2 = {static_cast<float32_t>(erf(ar.values[4])), static_cast<float32_t>(erf(ar.values[5]))};
273
+ float32x2_t er_vec3 = {static_cast<float32_t>(erf(ar.values[6])), static_cast<float32_t>(erf(ar.values[7]))};
274
+
275
+ float32x4_t result0 = vcombine_f32(er_vec0, er_vec1);
276
+ float32x4_t result1 = vcombine_f32(er_vec2, er_vec3);
277
+
278
+ float32x4x2_t result;
279
+ result.val[0] = result0;
280
+ result.val[1] = result1;
281
+
282
+ return FP32Vec8(result);
283
+ }
284
+
285
+ FP32Vec8 operator*(const FP32Vec8 &b) const {
286
+ return FP32Vec8(float32x4x2_t({vmulq_f32(reg.val[0], b.reg.val[0]), vmulq_f32(reg.val[1], b.reg.val[1])}));
287
+ }
288
+
289
+ FP32Vec8 operator+(const FP32Vec8 &b) const {
290
+ return FP32Vec8(float32x4x2_t({vaddq_f32(reg.val[0], b.reg.val[0]), vaddq_f32(reg.val[1], b.reg.val[1])}));
291
+ }
292
+
293
+ FP32Vec8 operator-(const FP32Vec8 &b) const {
294
+ return FP32Vec8(float32x4x2_t({vsubq_f32(reg.val[0], b.reg.val[0]), vsubq_f32(reg.val[1], b.reg.val[1])}));
295
+ }
296
+
297
+ FP32Vec8 operator/(const FP32Vec8 &b) const {
298
+ return FP32Vec8(float32x4x2_t({vdivq_f32(reg.val[0], b.reg.val[0]), vdivq_f32(reg.val[1], b.reg.val[1])}));
299
+ }
300
+
301
+ void save(float *ptr) const {
302
+ vst1q_f32(ptr, reg.val[0]);
303
+ vst1q_f32(ptr + 4, reg.val[1]);
304
+ }
305
+};
306
+
307
+struct FP32Vec16 : public Vec<FP32Vec16> {
308
+ constexpr static int VEC_ELEM_NUM = 16;
309
+ union AliasReg {
310
+ float32x4x4_t reg;
311
+ float values[VEC_ELEM_NUM];
312
+ };
313
+
314
+ float32x4x4_t reg;
315
+
316
+ explicit FP32Vec16(float v) : reg({vmovq_n_f32(v), vmovq_n_f32(v), vmovq_n_f32(v), vmovq_n_f32(v)}) {}
317
+
318
+ explicit FP32Vec16() : reg({vmovq_n_f32(0.0), vmovq_n_f32(0.0), vmovq_n_f32(0.0), vmovq_n_f32(0.0)}) {}
319
+
320
+ explicit FP32Vec16(const float *ptr) : reg({vld1q_f32(ptr), vld1q_f32(ptr + 4), vld1q_f32(ptr + 8), vld1q_f32(ptr + 12)}) {}
321
+
322
+ explicit FP32Vec16(float32x4x4_t data) : reg(data) {}
323
+
324
+ explicit FP32Vec16(const FP32Vec8 &data) {
325
+ reg.val[0] = data.reg.val[0];
326
+ reg.val[1] = data.reg.val[1];
327
+ reg.val[2] = data.reg.val[0];
328
+ reg.val[3] = data.reg.val[1];
329
+ }
330
+
331
+ explicit FP32Vec16(const FP32Vec16 &data) : reg(data.reg) {}
332
+
333
+ explicit FP32Vec16(const FP16Vec8 &v) : FP32Vec16(FP32Vec8(v.reg)) {}
334
+
335
+ #ifdef ARM_BF16_SUPPORT
336
+ explicit FP32Vec16(bfloat16x8x2_t v) : reg({
337
+ vcvtq_low_f32_bf16(v.val[0]),
338
+ vcvtq_high_f32_bf16(v.val[0]),
339
+ vcvtq_low_f32_bf16(v.val[1]),
340
+ vcvtq_high_f32_bf16(v.val[1])
341
+ }) {};
342
+ #endif
343
+
344
+ explicit FP32Vec16(const FP32Vec4 &data) {
345
+ reg.val[0] = data.reg;
346
+ reg.val[1] = data.reg;
347
+ reg.val[2] = data.reg;
348
+ reg.val[3] = data.reg;
349
+ };
350
+
351
+ #ifdef ARM_BF16_SUPPORT
352
+ explicit FP32Vec16(const BF16Vec16 &v) : reg({
353
+ vcvtq_low_f32_bf16(v.reg.val[0]),
354
+ vcvtq_high_f32_bf16(v.reg.val[0]),
355
+ vcvtq_low_f32_bf16(v.reg.val[1]),
356
+ vcvtq_high_f32_bf16(v.reg.val[1])
357
+ }) {};
358
+
359
+ explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {};
360
+ #endif
361
+
362
+ explicit FP32Vec16(const FP16Vec16 &v) {
363
+ reg.val[0] = vcvt_f32_f16(vget_low_f16(v.reg.val[0]));
364
+ reg.val[1] = vcvt_f32_f16(vget_high_f16(v.reg.val[0]));
365
+ reg.val[2] = vcvt_f32_f16(vget_low_f16(v.reg.val[1]));
366
+ reg.val[3] = vcvt_f32_f16(vget_high_f16(v.reg.val[1]));
367
+ };
368
+
369
+ FP32Vec16 operator+(const FP32Vec16 &b) const {
370
+ return FP32Vec16(float32x4x4_t({
371
+ vaddq_f32(reg.val[0], b.reg.val[0]),
372
+ vaddq_f32(reg.val[1], b.reg.val[1]),
373
+ vaddq_f32(reg.val[2], b.reg.val[2]),
374
+ vaddq_f32(reg.val[3], b.reg.val[3])}));
375
+ };
376
+
377
+ FP32Vec16 operator*(const FP32Vec16 &b) const {
378
+ return FP32Vec16(float32x4x4_t({
379
+ vmulq_f32(reg.val[0], b.reg.val[0]),
380
+ vmulq_f32(reg.val[1], b.reg.val[1]),
381
+ vmulq_f32(reg.val[2], b.reg.val[2]),
382
+ vmulq_f32(reg.val[3], b.reg.val[3])}));
383
+ };
384
+
385
+ FP32Vec16 operator-(const FP32Vec16 &b) const {
386
+ return FP32Vec16(float32x4x4_t({
387
+ vsubq_f32(reg.val[0], b.reg.val[0]),
388
+ vsubq_f32(reg.val[1], b.reg.val[1]),
389
+ vsubq_f32(reg.val[2], b.reg.val[2]),
390
+ vsubq_f32(reg.val[3], b.reg.val[3])
391
+ }));
392
+ };
393
+
394
+ FP32Vec16 operator/(const FP32Vec16 &b) const {
395
+ return FP32Vec16(float32x4x4_t({
396
+ vdivq_f32(reg.val[0], b.reg.val[0]),
397
+ vdivq_f32(reg.val[1], b.reg.val[1]),
398
+ vdivq_f32(reg.val[2], b.reg.val[2]),
399
+ vdivq_f32(reg.val[3], b.reg.val[3])
400
+ }));
401
+ };
402
+
403
+ float reduce_sum() const {
404
+ AliasReg ar;
405
+ ar.reg = reg;
406
+ float answer = 0;
407
+ unroll_loop<int, VEC_ELEM_NUM>([&answer, &ar](int i) { answer += ar.values[i]; });
408
+
409
+ return answer;
410
+ };
411
+
412
+ template <int group_size> float reduce_sub_sum(int idx) {
413
+ static_assert(VEC_ELEM_NUM % group_size == 0);
414
+
415
+ AliasReg ar;
416
+ ar.reg = reg;
417
+ float answer = 0;
418
+ const int start = idx * group_size;
419
+ unroll_loop<int, group_size>(
420
+ [&answer, &start, ar](int i) { answer += ar.values[start + i]; });
421
+
422
+ return answer;
423
+ };
424
+
425
+ void save(float *ptr) const {
426
+ vst1q_f32(ptr, reg.val[0]);
427
+ vst1q_f32(ptr + 4, reg.val[1]);
428
+ vst1q_f32(ptr + 8, reg.val[2]);
429
+ vst1q_f32(ptr + 12, reg.val[3]);
430
+ };
431
+};
432
+
433
+template <typename T> struct VecType { using vec_type = void; };
434
+
435
+template <typename T> using vec_t = typename VecType<T>::vec_type;
436
+
437
+template <> struct VecType<float> { using vec_type = FP32Vec8; };
438
+
439
+template <> struct VecType<c10::Half> { using vec_type = FP16Vec8; };
440
+
441
+#ifdef ARM_BF16_SUPPORT
442
+template <> struct VecType<c10::BFloat16> { using vec_type = BF16Vec8; };
443
+#endif
444
+
445
+template <typename T> void storeFP32(float v, T *ptr) { *ptr = v; }
446
+
447
+template <> inline void storeFP32<c10::Half>(float v, c10::Half *ptr) {
448
+ *reinterpret_cast<__fp16 *>(ptr) = v;
449
+}
450
+
451
+inline FP16Vec16::FP16Vec16(const FP32Vec16 &v) {
452
+ float16x4_t low_0 = vcvt_f16_f32(v.reg.val[0]);
453
+ float16x4_t high_0 = vcvt_f16_f32(v.reg.val[1]);
454
+ float16x4_t low_1 = vcvt_f16_f32(v.reg.val[2]);
455
+ float16x4_t high_1 = vcvt_f16_f32(v.reg.val[3]);
456
+
457
+ reg.val[0] = vcombine_f16(low_0, high_0);
458
+ reg.val[1] = vcombine_f16(low_1, high_1);
459
+};
460
+
461
+inline FP16Vec8 :: FP16Vec8(const FP32Vec8 &v) {
462
+ float16x4_t lower_half = vcvt_f16_f32(v.reg.val[0]);
463
+ float16x4_t upper_half = vcvt_f16_f32(v.reg.val[1]);
464
+
465
+ reg = vcombine_f16(lower_half, upper_half);
466
+};
467
+
468
+inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
469
+
470
+ acc.reg.val[0] = vfmaq_f32(acc.reg.val[0], a.reg.val[0], b.reg.val[0]);
471
+ acc.reg.val[1] = vfmaq_f32(acc.reg.val[1], a.reg.val[1], b.reg.val[1]);
472
+ acc.reg.val[2] = vfmaq_f32(acc.reg.val[2], a.reg.val[2], b.reg.val[2]);
473
+ acc.reg.val[3] = vfmaq_f32(acc.reg.val[3], a.reg.val[3], b.reg.val[3]);
474
+};
475
+
476
+#ifdef ARM_BF16_SUPPORT
477
+inline void fma(FP32Vec16 &acc, BF16Vec32 &a, BF16Vec32 &b) {
478
+
479
+ float32x4_t a0_low = vcvt_f32_bf16(vget_low_bf16(a.reg.val[0]));
480
+ float32x4_t a0_high = vcvt_f32_bf16(vget_high_bf16(a.reg.val[0]));
481
+ float32x4_t a1_low = vcvt_f32_bf16(vget_low_bf16(a.reg.val[1]));
482
+ float32x4_t a1_high = vcvt_f32_bf16(vget_high_bf16(a.reg.val[1]));
483
+
484
+ float32x4_t b0_low = vcvt_f32_bf16(vget_low_bf16(b.reg.val[0]));
485
+ float32x4_t b0_high = vcvt_f32_bf16(vget_high_bf16(b.reg.val[0]));
486
+ float32x4_t b1_low = vcvt_f32_bf16(vget_low_bf16(b.reg.val[1]));
487
+ float32x4_t b1_high = vcvt_f32_bf16(vget_high_bf16(b.reg.val[1]));
488
+
489
+ acc.reg.val[0] = vfmaq_f32(acc.reg.val[0], a0_low, b0_low);
490
+ acc.reg.val[1] = vfmaq_f32(acc.reg.val[1], a0_high, b0_high);
491
+ acc.reg.val[2] = vfmaq_f32(acc.reg.val[2], a1_low, b1_low);
492
+ acc.reg.val[3] = vfmaq_f32(acc.reg.val[3], a1_high, b1_high);
493
+};
494
+#endif
495
+
496
+#ifdef ARM_BF16_SUPPORT
497
+inline BF16Vec8::BF16Vec8(const FP32Vec8 &v) : reg(vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.reg.val[0]), v.reg.val[1])) {};
498
+
499
+inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) : reg({
500
+ vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.reg.val[0]), v.reg.val[1]),
501
+ vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.reg.val[2]), v.reg.val[3])
502
+ }){};
503
+#endif
504
+
505
+inline void prefetch(const void *addr) {
506
+ __builtin_prefetch(addr, 0, 1);
507
+};
508
+
509
+#ifdef ARM_BF16_SUPPORT
510
+template <>
511
+inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
512
+ *reinterpret_cast<__bf16 *>(ptr) = vcvth_bf16_f32(v);
513
+};
514
+#endif
515
+};
\ No newline at end of file
docs/source/getting_started/arm-installation.rst
new
+50
@@ -0,0 +1,50 @@
1
+.. _installation_arm:
2
+
3
+Installation for ARM CPUs
4
+=========================
5
+
6
+vLLM has been adapted to work on ARM64 CPUs with NEON support, leveraging the CPU backend initially developed for the x86 platform. This guide provides installation instructions specific to ARM. For additional details on supported features, refer to the x86 platform documentation covering:
7
+
8
+* CPU backend inference capabilities
9
+* Relevant runtime environment variables
10
+* Performance optimization tips
11
+
12
+ARM CPU backend currently supports Float32, FP16 and BFloat16 datatypes.
13
+Contents:
14
+
15
+1. :ref:`Requirements <arm_backend_requirements>`
16
+2. :ref:`Quick Start with Dockerfile <arm_backend_quick_start_dockerfile>`
17
+3. :ref:`Building from Source <build_arm_backend_from_source>`
18
+
19
+.. _arm_backend_requirements:
20
+
21
+Requirements
22
+------------
23
+
24
+* **Operating System**: Linux or macOS
25
+* **Compiler**: gcc/g++ >= 12.3.0 (optional, but recommended)
26
+* **Instruction Set Architecture (ISA)**: NEON support is required
27
+
28
+.. _arm_backend_quick_start_dockerfile:
29
+
30
+Quick Start with Dockerfile
31
+---------------------------
32
+
33
+You can quickly set up vLLM on ARM using Docker:
34
+
35
+.. code-block:: console
36
+
37
+ $ docker build -f Dockerfile.arm -t vllm-cpu-env --shm-size=4g .
38
+ $ docker run -it \
39
+ --rm \
40
+ --network=host \
41
+ --cpuset-cpus=<cpu-id-list, optional> \
42
+ --cpuset-mems=<memory-node, optional> \
43
+ vllm-cpu-env
44
+
45
+.. _build_arm_backend_from_source:
46
+
47
+Building from Source
48
+--------------------
49
+
50
+To build vLLM from source on Ubuntu 22.04 or other Linux distributions, follow a similar process as with x86. Testing has been conducted on AWS Graviton3 instances for compatibility.
docs/source/index.rst
+1
@@ -67,6 +67,7 @@ Documentation
67
getting_started/openvino-installation
68
getting_started/cpu-installation
69
getting_started/gaudi-installation
70
+ getting_started/arm-installation
71
getting_started/neuron-installation
72
getting_started/tpu-installation
73
getting_started/xpu-installation
examples/offline_inference.py
+1
-1
@@ -19,4 +19,4 @@ outputs = llm.generate(prompts, sampling_params)
19
for output in outputs:
20
prompt = output.prompt
21
generated_text = output.outputs[0].text
22
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
22
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
\ No newline at end of file
requirements-cpu.txt
+4
-3
@@ -1,6 +1,7 @@
1
# Common dependencies
2
-r requirements-common.txt
3
4
-# Dependencies for x86_64 CPUs
5
-torch == 2.5.1+cpu; platform_machine != "ppc64le"
6
-torchvision; platform_machine != "ppc64le" # required for the image processor of phi3v, this must be updated alongside torch
4
+# Dependencies for CPUs
5
+torch==2.5.1+cpu; platform_machine != "ppc64le" and platform_machine != "aarch64"
6
+torch==2.5.1; platform_machine == "aarch64"
7
+torchvision; platform_machine != "ppc64le" # required for the image processor of phi3v, this must be updated alongside torch
\ No newline at end of file