@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 38a1674abb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

Support CPU inference with VSX PowerPC ISA (#5652)

Chip Kerchner committed Jun 26, 2024 at 17:53 UTC 38a1674abbba38344543170cb552e88e7f619167
7 files changed +1049 -511
Dockerfile.ppc64le new
+22
@@ -0,0 +1,22 @@
1 +FROM mambaorg/micromamba
2 +ARG MAMBA_DOCKERFILE_ACTIVATE=1
3 +USER root
4 +
5 +RUN apt-get update -y && apt-get install -y git wget vim numactl gcc-12 g++-12 protobuf-compiler libprotobuf-dev && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
6 +
7 +# Some packages in requirements-cpu are installed here
8 +# IBM provides optimized packages for ppc64le processors in the open-ce project for mamba
9 +# Currently these may not be available for venv or pip directly
10 +RUN micromamba install -y -n base -c https://ftp.osuosl.org/pub/open-ce/1.11.0-p10/ -c defaults python=3.10 pytorch-cpu=2.1.2 torchvision-cpu=0.16.2 && micromamba clean --all --yes
11 +
12 +COPY ./ /workspace/vllm
13 +
14 +WORKDIR /workspace/vllm
15 +
16 +# These packages will be in rocketce eventually
17 +RUN pip install -v -r requirements-cpu.txt --prefer-binary --extra-index-url https://repo.fury.io/mgiessing
18 +
19 +RUN VLLM_TARGET_DEVICE=cpu python3 setup.py install
20 +
21 +WORKDIR /vllm-workspace
22 +ENTRYPOINT ["/opt/conda/bin/python3", "-m", "vllm.entrypoints.openai.api_server"]
cmake/cpu_extension.cmake
+10 -1
@@ -46,6 +46,8 @@ is_avx512_disabled(AVX512_DISABLED)
46
47 find_isa(${CPUINFO} "avx2" AVX2_FOUND)
48 find_isa(${CPUINFO} "avx512f" AVX512_FOUND)
49 +find_isa(${CPUINFO} "POWER10" POWER10_FOUND)
50 +find_isa(${CPUINFO} "POWER9" POWER9_FOUND)
51
52 if (AVX512_FOUND AND NOT AVX512_DISABLED)
53 list(APPEND CXX_COMPILE_FLAGS
@@ -68,8 +70,15 @@ if (AVX512_FOUND AND NOT AVX512_DISABLED)
70 elseif (AVX2_FOUND)
71 list(APPEND CXX_COMPILE_FLAGS "-mavx2")
72 message(WARNING "vLLM CPU backend using AVX2 ISA")
73 +elseif (POWER9_FOUND OR POWER10_FOUND)
74 + message(STATUS "PowerPC detected")
75 + # Check for PowerPC VSX support
76 + list(APPEND CXX_COMPILE_FLAGS
77 + "-mvsx"
78 + "-mcpu=native"
79 + "-mtune=native")
80 else()
72 - message(FATAL_ERROR "vLLM CPU backend requires AVX512 or AVX2 ISA support.")
81 + message(FATAL_ERROR "vLLM CPU backend requires AVX512 or AVX2 or Power9+ ISA support.")
82 endif()
83
84 message(STATUS "CPU extension compile flags: ${CXX_COMPILE_FLAGS}")
csrc/cpu/cpu_types.hpp
+7 -507
@@ -2,514 +2,14 @@
2 #ifndef CPU_TYPES_HPP
3 #define CPU_TYPES_HPP
4
5 -#include <immintrin.h>
6 -#include <torch/all.h>
7 -
8 -#ifndef __AVX2__
9 -static_assert(false, "AVX2 must be supported for the current implementation.");
10 -#endif
11 -
12 -namespace vec_op {
13 -
14 -// FIXME: FP16 is not fully supported in Torch-CPU
15 -#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
16 - AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
17 - AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
18 -
19 -#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
20 - AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
21 -
22 -#ifndef CPU_OP_GUARD
23 -#define CPU_KERNEL_GUARD_IN(NAME)
24 -#define CPU_KERNEL_GUARD_OUT(NAME)
25 -#else
26 -#define CPU_KERNEL_GUARD_IN(NAME) \
27 - std::cout << #NAME << " invoked." << std::endl;
28 -#define CPU_KERNEL_GUARD_OUT(NAME) std::cout << #NAME << " exit." << std::endl;
29 -#endif
30 -
31 -#define FORCE_INLINE __attribute__((always_inline)) inline
32 -
33 -namespace {
34 -template <typename T, T... indexes, typename F>
35 -constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F &&f) {
36 - (f(std::integral_constant<T, indexes>{}), ...);
37 -}
38 -}; // namespace
39 -
40 -template <typename T, T count, typename F,
41 - typename = std::enable_if_t<std::is_invocable_v<F, T>>>
42 -constexpr void unroll_loop(F &&f) {
43 - unroll_loop_item(std::make_integer_sequence<T, count>{}, std::forward<F>(f));
44 -}
45 -
46 -template <typename T> struct Vec {
47 - constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; }
48 -};
49 -
50 -struct FP32Vec8;
51 -struct FP32Vec16;
52 -
53 -#ifdef __AVX512FP16__
54 -struct FP16Vec8 : public Vec<FP16Vec8> {
55 - constexpr static int VEC_ELEM_NUM = 8;
56 -
57 - __m128h reg;
58 -
59 - explicit FP16Vec8(_Float16 v) : reg(_mm_set1_ph(v)) {}
60 -
61 - explicit FP16Vec8(const void *ptr) : reg(_mm_loadu_ph(ptr)) {}
62 -
63 - explicit FP16Vec8(__m128h data) : reg(data) {}
64 -
65 - FP16Vec8 operator*(const FP16Vec8 &b) const {
66 - return FP16Vec8(_mm_mul_ph(reg, b.reg));
67 - }
68 -
69 - FP16Vec8 operator+(const FP16Vec8 &b) const {
70 - return FP16Vec8(_mm_add_ph(reg, b.reg));
71 - }
72 -
73 - FP16Vec8 operator-(const FP16Vec8 &b) const {
74 - return FP16Vec8(_mm_sub_ph(reg, b.reg));
75 - }
76 -
77 - FP16Vec8 operator/(const FP16Vec8 &b) const {
78 - return FP16Vec8(_mm_div_ph(reg, b.reg));
79 - }
80 -
81 - void save(void *ptr) const { _mm_storeu_ph(ptr, reg); }
82 -};
83 -#endif
84 -
85 -struct BF16Vec8 : public Vec<BF16Vec8> {
86 - constexpr static int VEC_ELEM_NUM = 8;
87 -
88 - __m128i reg;
89 -
90 - explicit BF16Vec8(const void *ptr)
91 - : reg((__m128i)_mm_loadu_si128((__m128i *)ptr)) {}
92 -
93 - explicit BF16Vec8(const FP32Vec8 &);
94 -
95 - void save(void *ptr) const { *reinterpret_cast<__m128i *>(ptr) = reg; }
96 -};
97 -
98 -struct BF16Vec16 : public Vec<BF16Vec16> {
99 - constexpr static int VEC_ELEM_NUM = 16;
100 -
101 - __m256i reg;
102 -
103 - explicit BF16Vec16(const void *ptr)
104 - : reg((__m256i)_mm256_loadu_si256((__m256i *)ptr)) {}
105 -
106 - explicit BF16Vec16(const FP32Vec16 &);
107 -
108 - void save(void *ptr) const { *reinterpret_cast<__m256i *>(ptr) = reg; }
109 -};
110 -
111 -#ifdef __AVX512F__
112 -struct BF16Vec32 : public Vec<BF16Vec32> {
113 - constexpr static int VEC_ELEM_NUM = 32;
114 -
115 - __m512i reg;
116 -
117 - explicit BF16Vec32(const void *ptr) : reg((__m512i)_mm512_loadu_si512(ptr)) {}
118 -
119 - explicit BF16Vec32(__m512i data) : reg(data) {}
120 -
121 - explicit BF16Vec32(BF16Vec8 &vec8_data)
122 - : reg((__m512i)_mm512_inserti32x4(
123 - _mm512_inserti32x4(_mm512_inserti32x4(_mm512_castsi128_si512(
124 - (__m128i)vec8_data.reg),
125 - (__m128i)vec8_data.reg, 1),
126 - (__m128i)vec8_data.reg, 2),
127 - (__m128i)vec8_data.reg, 3)) {}
128 -
129 - void save(void *ptr) const { *reinterpret_cast<__m512i *>(ptr) = reg; }
130 -};
131 -#else
132 -struct BF16Vec32 : public Vec<BF16Vec32> {
133 - constexpr static int VEC_ELEM_NUM = 32;
134 -
135 - __m256i reg_low;
136 - __m256i reg_high;
137 -
138 - explicit BF16Vec32(const void *ptr)
139 - : reg_low(_mm256_loadu_si256((__m256i const *)ptr)),
140 - reg_high(_mm256_loadu_si256((__m256i const *)ptr + 1)) {}
141 -
142 - explicit BF16Vec32(__m256i low, __m256i high) : reg_low(low),
143 - reg_high(high) {}
144 -
145 - explicit BF16Vec32(BF16Vec8 &vec8_data)
146 - : reg_low((__m256i)_mm256_inserti32x4(
147 - _mm256_castsi128_si256((__m128i)vec8_data.reg),
148 - (__m128i)vec8_data.reg, 1)),
149 - reg_high((__m256i)_mm256_inserti32x4(
150 - _mm256_castsi128_si256((__m128i)vec8_data.reg),
151 - (__m128i)vec8_data.reg, 1)) {}
152 -
153 - void save(void *ptr) const {
154 - *reinterpret_cast<__m256i *>(ptr) = reg_low;
155 - *reinterpret_cast<__m256i *>((__m256i *)ptr + 1) = reg_high;
156 - }
157 -};
158 -#endif
159 -
160 -struct FP32Vec4 : public Vec<FP32Vec4> {
161 - constexpr static int VEC_ELEM_NUM = 4;
162 - union AliasReg {
163 - __m128 reg;
164 - float values[VEC_ELEM_NUM];
165 - };
166 -
167 - __m128 reg;
168 -
169 - explicit FP32Vec4(float v) : reg(_mm_set1_ps(v)) {}
170 -
171 - explicit FP32Vec4() : reg(_mm_set1_ps(0.0)) {}
172 -
173 - explicit FP32Vec4(const float *ptr) : reg(_mm_loadu_ps(ptr)) {}
174 -
175 - explicit FP32Vec4(__m128 data) : reg(data) {}
176 -
177 - explicit FP32Vec4(const FP32Vec4 &data) : reg(data.reg) {}
178 -};
179 -
180 -struct FP32Vec8 : public Vec<FP32Vec8> {
181 - constexpr static int VEC_ELEM_NUM = 8;
182 - union AliasReg {
183 - __m256 reg;
184 - float values[VEC_ELEM_NUM];
185 - };
186 -
187 - __m256 reg;
188 -
189 - explicit FP32Vec8(float v) : reg(_mm256_set1_ps(v)) {}
190 -
191 - explicit FP32Vec8() : reg(_mm256_set1_ps(0.0)) {}
192 -
193 - explicit FP32Vec8(const float *ptr) : reg(_mm256_loadu_ps(ptr)) {}
194 -
195 - explicit FP32Vec8(__m256 data) : reg(data) {}
196 -
197 - explicit FP32Vec8(const FP32Vec8 &data) : reg(data.reg) {}
198 -
199 -#ifdef __AVX512FP16__
200 - explicit FP32Vec8(__m128h v) : reg(_mm256_cvtph_ps(_mm_castph_si128(v))) {}
201 -#endif
202 -
203 - explicit FP32Vec8(const BF16Vec8 &v)
204 - : reg(_mm256_castsi256_ps(
205 - _mm256_bslli_epi128(_mm256_cvtepu16_epi32(v.reg), 2))) {}
206 -
207 - float reduce_sum() const {
208 - AliasReg ar;
209 - ar.reg = reg;
210 - float result = 0;
211 - unroll_loop<int, VEC_ELEM_NUM>([&result, &ar](int i) { result += ar.values[i]; });
212 -
213 - return result;
214 - }
215 -
216 - FP32Vec8 exp() const {
217 - AliasReg ar;
218 - ar.reg = reg;
219 - return FP32Vec8(_mm256_set_ps(expf(ar.values[7]), expf(ar.values[6]),
220 - expf(ar.values[5]), expf(ar.values[4]),
221 - expf(ar.values[3]), expf(ar.values[2]),
222 - expf(ar.values[1]), expf(ar.values[0])));
223 - }
224 -
225 - FP32Vec8 tanh() const {
226 - AliasReg ar;
227 - ar.reg = reg;
228 - return FP32Vec8(_mm256_set_ps(tanhf(ar.values[7]), tanhf(ar.values[6]),
229 - tanhf(ar.values[5]), tanhf(ar.values[4]),
230 - tanhf(ar.values[3]), tanhf(ar.values[2]),
231 - tanhf(ar.values[1]), tanhf(ar.values[0])));
232 - }
233 -
234 - FP32Vec8 er() const {
235 - AliasReg ar;
236 - ar.reg = reg;
237 - return FP32Vec8(_mm256_set_ps(erf(ar.values[7]), erf(ar.values[6]),
238 - erf(ar.values[5]), erf(ar.values[4]),
239 - erf(ar.values[3]), erf(ar.values[2]),
240 - erf(ar.values[1]), erf(ar.values[0])));
241 - }
242 -
243 - FP32Vec8 operator*(const FP32Vec8 &b) const {
244 - return FP32Vec8(_mm256_mul_ps(reg, b.reg));
245 - }
246 -
247 - FP32Vec8 operator+(const FP32Vec8 &b) const {
248 - return FP32Vec8(_mm256_add_ps(reg, b.reg));
249 - }
250 -
251 - FP32Vec8 operator-(const FP32Vec8 &b) const {
252 - return FP32Vec8(_mm256_sub_ps(reg, b.reg));
253 - }
254 -
255 - FP32Vec8 operator/(const FP32Vec8 &b) const {
256 - return FP32Vec8(_mm256_div_ps(reg, b.reg));
257 - }
258 -
259 - void save(float *ptr) const { _mm256_storeu_ps(ptr, reg); }
260 -};
261 -
262 -#ifdef __AVX512F__
263 -struct FP32Vec16 : public Vec<FP32Vec16> {
264 - constexpr static int VEC_ELEM_NUM = 16;
265 - union AliasReg {
266 - __m512 reg;
267 - float values[VEC_ELEM_NUM];
268 - };
269 -
270 - __m512 reg;
271 -
272 - explicit FP32Vec16(float v) : reg(_mm512_set1_ps(v)) {}
273 -
274 - explicit FP32Vec16() : reg(_mm512_set1_ps(0.0)) {}
275 -
276 - explicit FP32Vec16(const float *ptr) : reg(_mm512_loadu_ps(ptr)) {}
277 -
278 - explicit FP32Vec16(__m512 data) : reg(data) {}
279 -
280 - explicit FP32Vec16(const FP32Vec16 &data) : reg(data.reg) {}
281 -
282 - explicit FP32Vec16(const FP32Vec4 &data)
283 - : reg((__m512)_mm512_inserti32x4(
284 - _mm512_inserti32x4(
285 - _mm512_inserti32x4(_mm512_castsi128_si512((__m128i)data.reg),
286 - (__m128i)data.reg, 1),
287 - (__m128i)data.reg, 2),
288 - (__m128i)data.reg, 3)) {}
289 -
290 - explicit FP32Vec16(const FP32Vec8 &data)
291 - : reg((__m512)_mm512_inserti32x8(
292 - _mm512_castsi256_si512((__m256i)data.reg), (__m256i)data.reg, 1)) {}
293 -
294 - explicit FP32Vec16(const BF16Vec16 &v)
295 - : reg(_mm512_castsi512_ps(
296 - _mm512_bslli_epi128(_mm512_cvtepu16_epi32(v.reg), 2))) {}
297 -
298 - explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
299 -
300 - FP32Vec16 operator*(const FP32Vec16 &b) const {
301 - return FP32Vec16(_mm512_mul_ps(reg, b.reg));
302 - }
303 -
304 - FP32Vec16 operator+(const FP32Vec16 &b) const {
305 - return FP32Vec16(_mm512_add_ps(reg, b.reg));
306 - }
307 -
308 - FP32Vec16 operator-(const FP32Vec16 &b) const {
309 - return FP32Vec16(_mm512_sub_ps(reg, b.reg));
310 - }
311 -
312 - FP32Vec16 operator/(const FP32Vec16 &b) const {
313 - return FP32Vec16(_mm512_div_ps(reg, b.reg));
314 - }
315 -
316 - float reduce_sum() const { return _mm512_reduce_add_ps(reg); }
317 -
318 - template <int group_size> float reduce_sub_sum(int idx) {
319 - static_assert(VEC_ELEM_NUM % group_size == 0);
320 - constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
321 - __mmask16 mask = _cvtu32_mask16(base_mask << (idx * group_size));
322 - return _mm512_mask_reduce_add_ps(mask, reg);
323 - }
324 -
325 - void save(float *ptr) const { _mm512_storeu_ps(ptr, reg); }
326 -};
5 +#if defined(__x86_64__)
6 + //x86 implementation
7 + #include "cpu_types_x86.hpp"
8 +#elif defined(__POWER9_VECTOR__)
9 + //ppc implementation
10 + #include "cpu_types_vsx.hpp"
11 #else
328 -struct FP32Vec16 : public Vec<FP32Vec16> {
329 - constexpr static int VEC_ELEM_NUM = 16;
330 -
331 - union AliasReg {
332 - __m256 reg;
333 - float values[8];
334 - };
335 -
336 - __m256 reg_low;
337 - __m256 reg_high;
338 -
339 - explicit FP32Vec16(float v) : reg_low(_mm256_set1_ps(v)),
340 - reg_high(_mm256_set1_ps(v)) {}
341 -
342 - explicit FP32Vec16() : reg_low(_mm256_set1_ps(0.0)),
343 - reg_high(_mm256_set1_ps(0.0)) {}
344 -
345 - explicit FP32Vec16(const float *ptr) : reg_low(_mm256_loadu_ps(ptr)),
346 - reg_high(_mm256_loadu_ps(ptr + 8)) {}
347 -
348 - explicit FP32Vec16(__m256 low, __m256 high) : reg_low(low), reg_high(high) {}
349 -
350 - explicit FP32Vec16(const FP32Vec16 &data) : reg_low(data.reg_low),
351 - reg_high(data.reg_high) {}
352 -
353 - explicit FP32Vec16(const FP32Vec4 &data)
354 - : reg_low((__m256)_mm256_inserti128_si256(
355 - _mm256_castsi128_si256((__m128i)data.reg),
356 - (__m128i)data.reg, 1)),
357 - reg_high((__m256)_mm256_inserti128_si256(
358 - _mm256_castsi128_si256((__m128i)data.reg),
359 - (__m128i)data.reg, 1)) {}
360 -
361 - explicit FP32Vec16(const FP32Vec8 &data)
362 - : reg_low(data.reg), reg_high(data.reg) {}
363 -
364 - explicit FP32Vec16(const BF16Vec16 &v) {
365 - __m128i low = _mm256_extractf128_si256(v.reg, 0);
366 - __m128i high = _mm256_extractf128_si256(v.reg, 1);
367 -
368 - __m256i v_low_epi32 = _mm256_cvtepu16_epi32(low);
369 - __m256i v_high_epi32 = _mm256_cvtepu16_epi32(high);
370 -
371 - __m256i v_low_shifted = _mm256_bslli_epi128(v_low_epi32, 2);
372 - __m256i v_high_shifted = _mm256_bslli_epi128(v_high_epi32, 2);
373 -
374 - reg_low = _mm256_castsi256_ps(v_low_shifted);
375 - reg_high = _mm256_castsi256_ps(v_high_shifted);
376 - }
377 -
378 - explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
379 -
380 - FP32Vec16 operator*(const FP32Vec16 &b) const {
381 - return FP32Vec16(_mm256_mul_ps(reg_low, b.reg_low),
382 - _mm256_mul_ps(reg_high, b.reg_high));
383 - }
384 -
385 - FP32Vec16 operator+(const FP32Vec16 &b) const {
386 - return FP32Vec16(_mm256_add_ps(reg_low, b.reg_low),
387 - _mm256_add_ps(reg_high, b.reg_high));
388 - }
389 -
390 - FP32Vec16 operator-(const FP32Vec16 &b) const {
391 - return FP32Vec16(_mm256_sub_ps(reg_low, b.reg_low),
392 - _mm256_sub_ps(reg_high, b.reg_high));
393 - }
394 -
395 - FP32Vec16 operator/(const FP32Vec16 &b) const {
396 - return FP32Vec16(_mm256_div_ps(reg_low, b.reg_low),
397 - _mm256_div_ps(reg_high, b.reg_high));
398 - }
399 -
400 - float reduce_sum() const {
401 - FP32Vec8 low = FP32Vec8(reg_low);
402 - FP32Vec8 high = FP32Vec8(reg_high);
403 - return low.reduce_sum() + high.reduce_sum();
404 - }
405 -
406 - template <int group_size> float reduce_sub_sum(int idx) {
407 - float sum = 0.0;
408 - static_assert(VEC_ELEM_NUM % group_size == 0);
409 - constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
410 - uint32_t mask = base_mask << (idx * group_size);
411 -
412 - AliasReg ar;
413 -
414 - auto func = [&sum, &mask, &ar](int i) {
415 - int flag = mask & 0x1;
416 - mask = mask >> 1;
417 - if (flag != 0) sum += ar.values[i];
418 - };
419 -
420 - ar.reg = reg_low;
421 - unroll_loop<int, 8>(func);
422 -
423 - ar.reg = reg_high;
424 - unroll_loop<int, 8>(func);
425 -
426 - return sum;
427 - }
428 -
429 - void save(float *ptr) const {
430 - _mm256_storeu_ps(ptr, reg_low);
431 - _mm256_storeu_ps(ptr + 8, reg_high);
432 - }
433 -};
434 -#endif
435 -
436 -template <typename T> struct VecType { using vec_type = void; };
437 -
438 -template <typename T> using vec_t = typename VecType<T>::vec_type;
439 -
440 -template <> struct VecType<float> { using vec_type = FP32Vec8; };
441 -
442 -#ifdef __AVX512FP16__
443 -template <> struct VecType<c10::Half> { using vec_type = FP16Vec16; };
12 + #warning "unsupported vLLM cpu implementation"
13 #endif
14
446 -template <> struct VecType<c10::BFloat16> { using vec_type = BF16Vec8; };
447 -
448 -template <typename T> void storeFP32(float v, T *ptr) { *ptr = v; }
449 -
450 -#ifdef __AVX512FP16__
451 -template <> inline void storeFP32<c10::Half>(float v, c10::Half *ptr) {
452 - *reinterpret_cast<_Float16 *>(ptr) = v;
453 -}
454 -#endif
455 -
456 -inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
457 - acc = acc + a * b;
458 -}
459 -
460 -#ifdef __AVX512BF16__
461 -template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
462 - *reinterpret_cast<__bfloat16 *>(ptr) = _mm_cvtness_sbh(v);
463 -}
464 -
465 -inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
466 - : reg((__m128i)_mm256_cvtneps_pbh(v.reg)) {}
467 -
468 -inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
469 - : reg((__m256i)_mm512_cvtneps_pbh(v.reg)) {}
470 -
471 -inline void fma(FP32Vec16 &acc, BF16Vec32 &a, BF16Vec32 &b) {
472 - acc.reg = _mm512_dpbf16_ps(acc.reg, (__m512bh)a.reg, (__m512bh)b.reg);
473 -}
474 -#else
475 -template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
476 - c10::BFloat16 __attribute__((__may_alias__)) *v_ptr =
477 - reinterpret_cast<c10::BFloat16 *>(&v);
478 - *ptr = *(v_ptr + 1);
479 -}
480 -
481 -#ifdef __AVX512F__
482 -inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
483 - : reg(_mm256_cvtepi32_epi16(
484 - _mm256_bsrli_epi128(_mm256_castps_si256(v.reg), 2))) {}
485 -
486 -inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
487 - : reg(_mm512_cvtepi32_epi16(
488 - _mm512_bsrli_epi128(_mm512_castps_si512(v.reg), 2))) {}
489 -#else
490 -namespace{
491 -__m128i FP32Vec8_to_BF16Vec8_avx2(__m256 a) {
492 - __m256i ai = _mm256_castps_si256(a);
493 - ai = _mm256_srli_epi32(ai, 16);
494 - ai = _mm256_packus_epi32(ai, ai);
495 - ai = _mm256_permute4x64_epi64(ai, 0b00111001);
496 - return _mm256_extracti128_si256(ai, 0);
497 -}
498 -}
499 -
500 -inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
501 - : reg(FP32Vec8_to_BF16Vec8_avx2(v.reg)) {}
502 -
503 -inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
504 - BF16Vec8 low = BF16Vec8(FP32Vec8(v.reg_low));
505 - BF16Vec8 high = BF16Vec8(FP32Vec8(v.reg_high));
506 - reg = _mm256_insertf128_si256(_mm256_castsi128_si256(low.reg), high.reg, 1);
507 -}
508 -#endif // __AVX512F__
509 -#endif // __AVX512BF16__
510 -
511 -inline void prefetch(const void *addr) { _mm_prefetch(addr, _MM_HINT_T1); }
512 -
513 -}; // namespace vec_op
514 -
15 #endif
csrc/cpu/cpu_types_vsx.hpp new
+491
@@ -0,0 +1,491 @@
1 +
2 +#ifndef CPU_TYPES_VSX_HPP
3 +#define CPU_TYPES_VSX_HPP
4 +
5 +#include <altivec.h>
6 +#include <cmath>
7 +#include <torch/all.h>
8 +
9 +namespace vec_op {
10 +
11 +// FIXME: FP16 is not fully supported in Torch-CPU
12 +#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
13 + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
14 + AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
15 +
16 +#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
17 + AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
18 +
19 +#ifndef CPU_OP_GUARD
20 +#define CPU_KERNEL_GUARD_IN(NAME)
21 +#define CPU_KERNEL_GUARD_OUT(NAME)
22 +#else
23 +#define CPU_KERNEL_GUARD_IN(NAME) \
24 + std::cout << #NAME << " invoked." << std::endl;
25 +#define CPU_KERNEL_GUARD_OUT(NAME) std::cout << #NAME << " exit." << std::endl;
26 +#endif
27 +
28 +#define FORCE_INLINE __attribute__((always_inline)) inline
29 +
30 +namespace {
31 +template <typename T, T... indexes, typename F>
32 +constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F &&f) {
33 + (f(std::integral_constant<T, indexes>{}), ...);
34 +}
35 +}; // namespace
36 +
37 +template <typename T, T count, typename F,
38 + typename = std::enable_if_t<std::is_invocable_v<F, T>>>
39 +constexpr void unroll_loop(F &&f) {
40 + unroll_loop_item(std::make_integer_sequence<T, count>{}, std::forward<F>(f));
41 +}
42 +
43 +template <typename T> struct Vec {
44 + constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; }
45 +};
46 +
47 +typedef struct ss16x8x2_t {
48 + __vector signed short val[2];
49 +} ss16x8x2_t;
50 +
51 +typedef struct ss16x8x4_t {
52 + __vector signed short val[4];
53 +} ss16x8x4_t;
54 +
55 +typedef struct f32x4x2_t {
56 + __vector float val[2];
57 +} f32x4x2_t;
58 +
59 +typedef struct f32x4x4_t {
60 + __vector float val[4];
61 +} f32x4x4_t;
62 +
63 +struct FP32Vec8;
64 +struct FP32Vec16;
65 +
66 +struct BF16Vec8 : public Vec<BF16Vec8> {
67 + constexpr static int VEC_ELEM_NUM = 8;
68 +
69 + __vector signed short reg;
70 +
71 + explicit BF16Vec8(const void *ptr)
72 + : reg((__vector signed short)vec_xl(0, (__vector signed short *)ptr)) {}
73 +
74 + explicit BF16Vec8(const FP32Vec8 &);
75 +
76 + void save(void *ptr) const { *reinterpret_cast<__vector signed short *>(ptr) = reg; }
77 +};
78 +
79 +struct BF16Vec16 : public Vec<BF16Vec16> {
80 + constexpr static int VEC_ELEM_NUM = 16;
81 +
82 + ss16x8x2_t reg;
83 +
84 + explicit BF16Vec16(const void *ptr) {
85 + // Load 256 bits in two parts
86 + reg.val[0] = (__vector signed short)vec_xl(0, (signed short *)ptr);
87 + reg.val[1] = (__vector signed short)vec_xl(16, (signed short *)ptr);
88 + }
89 +
90 + explicit BF16Vec16(const FP32Vec16 &);
91 +
92 + void save(void *ptr) const {
93 + // Save 256 bits in two parts
94 + vec_xst(reg.val[0], 0, (signed short *)ptr);
95 + vec_xst(reg.val[1], 16, (signed short *)ptr);
96 + }
97 +};
98 +
99 +const static __vector signed short zero = vec_splats((signed short)0);
100 +
101 +struct BF16Vec32 : public Vec<BF16Vec32> {
102 + constexpr static int VEC_ELEM_NUM = 32;
103 +
104 + ss16x8x4_t reg;
105 + explicit BF16Vec32(const void *ptr)
106 + : reg(*reinterpret_cast<const ss16x8x4_t *>(ptr)) {}
107 +
108 + explicit BF16Vec32(ss16x8x4_t data) : reg(data) {}
109 +
110 + explicit BF16Vec32(const BF16Vec8 &vec8_data) : reg({
111 + vec8_data.reg,
112 + vec8_data.reg,
113 + vec8_data.reg,
114 + vec8_data.reg
115 + }) {}
116 +
117 + void save(void *ptr) const { *reinterpret_cast<ss16x8x4_t *>(ptr) = reg; }
118 +};
119 +
120 +struct FP32Vec4 : public Vec<FP32Vec4> {
121 + constexpr static int VEC_ELEM_NUM = 4;
122 + union AliasReg {
123 + __vector float reg;
124 + float values[VEC_ELEM_NUM];
125 + };
126 +
127 + __vector float reg;
128 +
129 + explicit FP32Vec4(float v) : reg(vec_splats(v)) {}
130 +
131 + explicit FP32Vec4() : reg(vec_splats(0.0f)) {}
132 +
133 + explicit FP32Vec4(const float *ptr) : reg(vec_xl(0, ptr)) {}
134 +
135 + explicit FP32Vec4(__vector float data) : reg(data) {}
136 +
137 + explicit FP32Vec4(const FP32Vec4 &data) : reg(data.reg) {}
138 +};
139 +
140 +struct FP32Vec8 : public Vec<FP32Vec8> {
141 + constexpr static int VEC_ELEM_NUM = 8;
142 + union AliasReg {
143 + f32x4x2_t reg;
144 + float values[VEC_ELEM_NUM];
145 + };
146 +
147 + f32x4x2_t reg;
148 +
149 + explicit FP32Vec8(float v) {
150 + reg.val[0] = vec_splats(v);
151 + reg.val[1] = vec_splats(v);
152 + }
153 +
154 + explicit FP32Vec8() {
155 + reg.val[0] = vec_splats(0.0f);
156 + reg.val[1] = vec_splats(0.0f);
157 + }
158 +
159 + explicit FP32Vec8(const float *ptr) {
160 + reg.val[0] = vec_xl(0, ptr);
161 + reg.val[1] = vec_xl(16, ptr);
162 + }
163 +
164 + explicit FP32Vec8(f32x4x2_t data) : reg(data) {}
165 +
166 + explicit FP32Vec8(const FP32Vec8 &data) {
167 + reg.val[0] = data.reg.val[0];
168 + reg.val[1] = data.reg.val[1];
169 + }
170 +
171 + explicit FP32Vec8(const BF16Vec8 &v) {
172 + reg.val[0] = (__vector float)vec_mergeh(zero, v.reg);
173 + reg.val[1] = (__vector float)vec_mergel(zero, v.reg);
174 + }
175 +
176 + float reduce_sum() const {
177 + AliasReg ar;
178 + ar.reg = reg;
179 + float result = 0;
180 + unroll_loop<int, VEC_ELEM_NUM>([&result, &ar](int i) { result += ar.values[i]; });
181 +
182 + return result;
183 + }
184 +
185 + FP32Vec8 exp() const {
186 + // TODO: Vectorize this
187 + AliasReg ar;
188 + ar.reg = reg;
189 + f32x4x4_t ret;
190 + ret.val[0][0] = std::exp(ar.values[0]);
191 + ret.val[0][1] = std::exp(ar.values[1]);
192 + ret.val[0][2] = std::exp(ar.values[2]);
193 + ret.val[0][3] = std::exp(ar.values[3]);
194 + ret.val[1][0] = std::exp(ar.values[4]);
195 + ret.val[1][1] = std::exp(ar.values[5]);
196 + ret.val[1][2] = std::exp(ar.values[6]);
197 + ret.val[1][3] = std::exp(ar.values[7]);
198 + return FP32Vec8(f32x4x2_t({ret.val[0], ret.val[1]}));
199 + }
200 +
201 + FP32Vec8 tanh() const {
202 + // TODO: Vectorize this
203 + AliasReg ar;
204 + ar.reg = reg;
205 + f32x4x4_t ret;
206 + ret.val[0][0] = std::tanh(ar.values[0]);
207 + ret.val[0][1] = std::tanh(ar.values[1]);
208 + ret.val[0][2] = std::tanh(ar.values[2]);
209 + ret.val[0][3] = std::tanh(ar.values[3]);
210 + ret.val[1][0] = std::tanh(ar.values[4]);
211 + ret.val[1][1] = std::tanh(ar.values[5]);
212 + ret.val[1][2] = std::tanh(ar.values[6]);
213 + ret.val[1][3] = std::tanh(ar.values[7]);
214 + return FP32Vec8(f32x4x2_t({ret.val[0], ret.val[1]}));
215 + }
216 +
217 + FP32Vec8 er() const {
218 + // TODO: Vectorize this
219 + AliasReg ar;
220 + ar.reg = reg;
221 + f32x4x4_t ret;
222 + ret.val[0][0] = std::erf(ar.values[0]);
223 + ret.val[0][1] = std::erf(ar.values[1]);
224 + ret.val[0][2] = std::erf(ar.values[2]);
225 + ret.val[0][3] = std::erf(ar.values[3]);
226 + ret.val[1][0] = std::erf(ar.values[4]);
227 + ret.val[1][1] = std::erf(ar.values[5]);
228 + ret.val[1][2] = std::erf(ar.values[6]);
229 + ret.val[1][3] = std::erf(ar.values[7]);
230 + return FP32Vec8(f32x4x2_t({ret.val[0], ret.val[1]}));
231 + }
232 +
233 + FP32Vec8 operator*(const FP32Vec8 &b) const {
234 + return FP32Vec8({vec_mul(reg.val[0], b.reg.val[0]), vec_mul(reg.val[1], b.reg.val[1])});
235 + }
236 +
237 + FP32Vec8 operator+(const FP32Vec8 &b) const {
238 + return FP32Vec8({vec_add(reg.val[0], b.reg.val[0]), vec_add(reg.val[1], b.reg.val[1])});
239 + }
240 +
241 + FP32Vec8 operator-(const FP32Vec8 &b) const {
242 + return FP32Vec8({vec_sub(reg.val[0], b.reg.val[0]), vec_sub(reg.val[1], b.reg.val[1])});
243 + }
244 +
245 + FP32Vec8 operator/(const FP32Vec8 &b) const {
246 + return FP32Vec8({vec_div(reg.val[0], b.reg.val[0]), vec_div(reg.val[1], b.reg.val[1])});
247 + }
248 +
249 + void save(float *ptr) const {
250 + vec_xst(reg.val[0], 0, ptr);
251 + vec_xst(reg.val[1], 16, ptr);
252 + }
253 +};
254 +
255 +struct FP32Vec16 : public Vec<FP32Vec16> {
256 + constexpr static int VEC_ELEM_NUM = 16;
257 + union AliasReg {
258 + f32x4x4_t reg;
259 + float values[VEC_ELEM_NUM];
260 + };
261 +
262 + f32x4x4_t reg;
263 +
264 + explicit FP32Vec16(float v) {
265 + reg.val[0] = vec_splats(v);
266 + reg.val[1] = vec_splats(v);
267 + reg.val[2] = vec_splats(v);
268 + reg.val[3] = vec_splats(v);
269 + }
270 +
271 + explicit FP32Vec16() {
272 + reg.val[0] = vec_splats(0.0f);
273 + reg.val[1] = vec_splats(0.0f);
274 + reg.val[2] = vec_splats(0.0f);
275 + reg.val[3] = vec_splats(0.0f);
276 + }
277 +
278 + explicit FP32Vec16(const float *ptr) {
279 + reg.val[0] = vec_xl(0, ptr);
280 + reg.val[1] = vec_xl(16, ptr);
281 + reg.val[2] = vec_xl(32, ptr);
282 + reg.val[3] = vec_xl(48, ptr);
283 + }
284 +
285 + explicit FP32Vec16(f32x4x4_t data) : reg(data) {}
286 +
287 + explicit FP32Vec16(const FP32Vec16 &data) {
288 + reg.val[0] = data.reg.val[0];
289 + reg.val[1] = data.reg.val[1];
290 + reg.val[2] = data.reg.val[2];
291 + reg.val[3] = data.reg.val[3];
292 + }
293 +
294 + explicit FP32Vec16(const FP32Vec4 &data) {
295 + reg.val[0] = data.reg;
296 + reg.val[1] = data.reg;
297 + reg.val[2] = data.reg;
298 + reg.val[3] = data.reg;
299 + }
300 +
301 + explicit FP32Vec16(const FP32Vec8 &data) {
302 + reg.val[0] = data.reg.val[0];
303 + reg.val[1] = data.reg.val[1];
304 + reg.val[2] = data.reg.val[0];
305 + reg.val[3] = data.reg.val[1];
306 + }
307 +
308 + explicit FP32Vec16(const BF16Vec16 &v) {
309 + reg.val[0] = (__vector float)vec_mergeh(zero, v.reg.val[0]);
310 + reg.val[1] = (__vector float)vec_mergel(zero, v.reg.val[0]);
311 + reg.val[2] = (__vector float)vec_mergeh(zero, v.reg.val[1]);
312 + reg.val[3] = (__vector float)vec_mergel(zero, v.reg.val[1]);
313 + }
314 +
315 + explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
316 +
317 + FP32Vec16 operator*(const FP32Vec16 &b) const {
318 + return FP32Vec16(f32x4x4_t({
319 + vec_mul(reg.val[0], b.reg.val[0]),
320 + vec_mul(reg.val[1], b.reg.val[1]),
321 + vec_mul(reg.val[2], b.reg.val[2]),
322 + vec_mul(reg.val[3], b.reg.val[3])}));
323 + }
324 +
325 + FP32Vec16 operator+(const FP32Vec16 &b) const {
326 + return FP32Vec16(f32x4x4_t({
327 + vec_add(reg.val[0], b.reg.val[0]),
328 + vec_add(reg.val[1], b.reg.val[1]),
329 + vec_add(reg.val[2], b.reg.val[2]),
330 + vec_add(reg.val[3], b.reg.val[3])}));
331 + }
332 +
333 + FP32Vec16 operator-(const FP32Vec16 &b) const {
334 + return FP32Vec16(f32x4x4_t({
335 + vec_sub(reg.val[0], b.reg.val[0]),
336 + vec_sub(reg.val[1], b.reg.val[1]),
337 + vec_sub(reg.val[2], b.reg.val[2]),
338 + vec_sub(reg.val[3], b.reg.val[3])}));
339 + }
340 +
341 + FP32Vec16 operator/(const FP32Vec16 &b) const {
342 + return FP32Vec16(f32x4x4_t({
343 + vec_div(reg.val[0], b.reg.val[0]),
344 + vec_div(reg.val[1], b.reg.val[1]),
345 + vec_div(reg.val[2], b.reg.val[2]),
346 + vec_div(reg.val[3], b.reg.val[3])}));
347 + }
348 +
349 + float reduce_sum() const {
350 + AliasReg ar;
351 + ar.reg = reg;
352 + float result = 0;
353 + unroll_loop<int, VEC_ELEM_NUM>([&result, &ar](int i) { result += ar.values[i]; });
354 +
355 + return result;
356 + }
357 +
358 + template <int group_size> float reduce_sub_sum(int idx) {
359 + static_assert(VEC_ELEM_NUM % group_size == 0);
360 +
361 + AliasReg ar;
362 + ar.reg = reg;
363 + float result = 0;
364 + const int start = idx * group_size;
365 + unroll_loop<int, group_size>(
366 + [&result, &start, ar](int i) { result += ar.values[start + i]; });
367 +
368 + return result;
369 + }
370 +
371 + void save(float *ptr) const {
372 + vec_xst(reg.val[0], 0, ptr);
373 + vec_xst(reg.val[1], 16, ptr);
374 + vec_xst(reg.val[2], 32, ptr);
375 + vec_xst(reg.val[3], 48, ptr);
376 + }
377 +};
378 +
379 +template <typename T> struct VecType { using vec_type = void; };
380 +
381 +template <typename T> using vec_t = typename VecType<T>::vec_type;
382 +
383 +template <> struct VecType<float> { using vec_type = FP32Vec8; };
384 +
385 +template <> struct VecType<c10::BFloat16> { using vec_type = BF16Vec8; };
386 +
387 +template <typename T> void storeFP32(float v, T *ptr) { *ptr = v; }
388 +
389 +inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
390 + acc = acc + a * b;
391 +}
392 +
393 +template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
394 + c10::BFloat16 __attribute__((__may_alias__)) *v_ptr =
395 + reinterpret_cast<c10::BFloat16 *>(&v);
396 + *ptr = *(v_ptr + 1);
397 +}
398 +
399 +#ifndef __VEC_CLASS_FP_NAN
400 +#define __VEC_CLASS_FP_NAN (1 << 6)
401 +#endif
402 +
403 +const static __vector unsigned char omask = { 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29 };
404 +#ifndef _ARCH_PWR10
405 +const static __vector unsigned int bias = { 0x00007fff, 0x00007fff, 0x00007fff, 0x00007fff };
406 +const static __vector unsigned int nan = { 0x7fc00000, 0x7fc00000, 0x7fc00000, 0x7fc00000 };
407 +const static __vector unsigned int sh16 = { 16, 16, 16, 16 };
408 +const static __vector unsigned int one = { 1, 1, 1, 1 };
409 +#endif
410 +
411 +inline BF16Vec8::BF16Vec8(const FP32Vec8 &v) {
412 +#ifdef _ARCH_PWR10
413 + __vector signed short ret[2];
414 + ret[0] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[0]);
415 + ret[1] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[1]);
416 + reg = vec_perm(ret[0], ret[1], omask);
417 +#elif defined(_ARCH_PWR9)
418 + __vector unsigned int inp0 = (__vector unsigned int)(v.reg.val[0]);
419 + __vector unsigned int inp1 = (__vector unsigned int)(v.reg.val[1]);
420 + __vector unsigned int lsb0 = vec_sr(inp0, sh16);
421 + __vector unsigned int lsb1 = vec_sr(inp1, sh16);
422 + lsb0 = vec_and(lsb0, one);
423 + lsb1 = vec_and(lsb1, one);
424 + __vector unsigned int rnd0 = vec_add(lsb0, bias);
425 + __vector unsigned int rnd1 = vec_add(lsb1, bias);
426 + inp0 = vec_add(inp0, rnd0);
427 + inp1 = vec_add(inp1, rnd1);
428 + __vector __bool int sel0 = vec_test_data_class(v.reg.val[0], __VEC_CLASS_FP_NAN);
429 + __vector __bool int sel1 = vec_test_data_class(v.reg.val[1], __VEC_CLASS_FP_NAN);
430 + inp0 = vec_sel(inp0, nan, sel0);
431 + inp1 = vec_sel(inp1, nan, sel1);
432 + inp0 = vec_sr(inp0, sh16);
433 + inp1 = vec_sr(inp1, sh16);
434 + reg = (__vector signed short)vec_perm(inp0, inp1, omask);
435 +#endif
436 +}
437 +
438 +inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
439 +#ifdef _ARCH_PWR10
440 + __vector signed short ret[4];
441 + ret[0] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[0]);
442 + ret[1] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[1]);
443 + ret[2] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[2]);
444 + ret[3] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[3]);
445 + reg.val[0] = vec_perm(ret[0], ret[1], omask);
446 + reg.val[1] = vec_perm(ret[2], ret[3], omask);
447 +#elif defined(_ARCH_PWR9)
448 + __vector unsigned int inp0 = (__vector unsigned int)(v.reg.val[0]);
449 + __vector unsigned int inp1 = (__vector unsigned int)(v.reg.val[1]);
450 + __vector unsigned int inp2 = (__vector unsigned int)(v.reg.val[2]);
451 + __vector unsigned int inp3 = (__vector unsigned int)(v.reg.val[3]);
452 + __vector unsigned int lsb0 = vec_sr(inp0, sh16);
453 + __vector unsigned int lsb1 = vec_sr(inp1, sh16);
454 + __vector unsigned int lsb2 = vec_sr(inp2, sh16);
455 + __vector unsigned int lsb3 = vec_sr(inp3, sh16);
456 + lsb0 = vec_and(lsb0, one);
457 + lsb1 = vec_and(lsb1, one);
458 + lsb2 = vec_and(lsb2, one);
459 + lsb3 = vec_and(lsb3, one);
460 + __vector unsigned int rnd0 = vec_add(lsb0, bias);
461 + __vector unsigned int rnd1 = vec_add(lsb1, bias);
462 + __vector unsigned int rnd2 = vec_add(lsb2, bias);
463 + __vector unsigned int rnd3 = vec_add(lsb3, bias);
464 + inp0 = vec_add(inp0, rnd0);
465 + inp1 = vec_add(inp1, rnd1);
466 + inp2 = vec_add(inp2, rnd2);
467 + inp3 = vec_add(inp3, rnd3);
468 + __vector __bool int sel0 = vec_test_data_class(v.reg.val[0], __VEC_CLASS_FP_NAN);
469 + __vector __bool int sel1 = vec_test_data_class(v.reg.val[1], __VEC_CLASS_FP_NAN);
470 + __vector __bool int sel2 = vec_test_data_class(v.reg.val[2], __VEC_CLASS_FP_NAN);
471 + __vector __bool int sel3 = vec_test_data_class(v.reg.val[3], __VEC_CLASS_FP_NAN);
472 + inp0 = vec_sel(inp0, nan, sel0);
473 + inp1 = vec_sel(inp1, nan, sel1);
474 + inp2 = vec_sel(inp2, nan, sel2);
475 + inp3 = vec_sel(inp3, nan, sel3);
476 + inp0 = vec_sr(inp0, sh16);
477 + inp1 = vec_sr(inp1, sh16);
478 + inp2 = vec_sr(inp2, sh16);
479 + inp3 = vec_sr(inp3, sh16);
480 + reg.val[0] = (__vector signed short)vec_perm(inp0, inp1, omask);
481 + reg.val[1] = (__vector signed short)vec_perm(inp2, inp3, omask);
482 +#endif
483 +}
484 +
485 +inline void prefetch(const void *addr) {
486 + __asm__ __volatile__("dcbt 0, %0" : : "r"(addr) : "memory");
487 +}
488 +
489 +}; // namespace vec_op
490 +
491 +#endif
csrc/cpu/cpu_types_x86.hpp new
+515
@@ -0,0 +1,515 @@
1 +
2 +#ifndef CPU_TYPES_X86_HPP
3 +#define CPU_TYPES_X86_HPP
4 +
5 +#include <immintrin.h>
6 +#include <torch/all.h>
7 +
8 +#ifndef __AVX2__
9 +static_assert(false, "AVX2 must be supported for the current implementation.");
10 +#endif
11 +
12 +namespace vec_op {
13 +
14 +// FIXME: FP16 is not fully supported in Torch-CPU
15 +#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
16 + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
17 + AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
18 +
19 +#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
20 + AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
21 +
22 +#ifndef CPU_OP_GUARD
23 +#define CPU_KERNEL_GUARD_IN(NAME)
24 +#define CPU_KERNEL_GUARD_OUT(NAME)
25 +#else
26 +#define CPU_KERNEL_GUARD_IN(NAME) \
27 + std::cout << #NAME << " invoked." << std::endl;
28 +#define CPU_KERNEL_GUARD_OUT(NAME) std::cout << #NAME << " exit." << std::endl;
29 +#endif
30 +
31 +#define FORCE_INLINE __attribute__((always_inline)) inline
32 +
33 +namespace {
34 +template <typename T, T... indexes, typename F>
35 +constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F &&f) {
36 + (f(std::integral_constant<T, indexes>{}), ...);
37 +}
38 +}; // namespace
39 +
40 +template <typename T, T count, typename F,
41 + typename = std::enable_if_t<std::is_invocable_v<F, T>>>
42 +constexpr void unroll_loop(F &&f) {
43 + unroll_loop_item(std::make_integer_sequence<T, count>{}, std::forward<F>(f));
44 +}
45 +
46 +template <typename T> struct Vec {
47 + constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; }
48 +};
49 +
50 +struct FP32Vec8;
51 +struct FP32Vec16;
52 +
53 +#ifdef __AVX512FP16__
54 +struct FP16Vec8 : public Vec<FP16Vec8> {
55 + constexpr static int VEC_ELEM_NUM = 8;
56 +
57 + __m128h reg;
58 +
59 + explicit FP16Vec8(_Float16 v) : reg(_mm_set1_ph(v)) {}
60 +
61 + explicit FP16Vec8(const void *ptr) : reg(_mm_loadu_ph(ptr)) {}
62 +
63 + explicit FP16Vec8(__m128h data) : reg(data) {}
64 +
65 + FP16Vec8 operator*(const FP16Vec8 &b) const {
66 + return FP16Vec8(_mm_mul_ph(reg, b.reg));
67 + }
68 +
69 + FP16Vec8 operator+(const FP16Vec8 &b) const {
70 + return FP16Vec8(_mm_add_ph(reg, b.reg));
71 + }
72 +
73 + FP16Vec8 operator-(const FP16Vec8 &b) const {
74 + return FP16Vec8(_mm_sub_ph(reg, b.reg));
75 + }
76 +
77 + FP16Vec8 operator/(const FP16Vec8 &b) const {
78 + return FP16Vec8(_mm_div_ph(reg, b.reg));
79 + }
80 +
81 + void save(void *ptr) const { _mm_storeu_ph(ptr, reg); }
82 +};
83 +#endif
84 +
85 +struct BF16Vec8 : public Vec<BF16Vec8> {
86 + constexpr static int VEC_ELEM_NUM = 8;
87 +
88 + __m128i reg;
89 +
90 + explicit BF16Vec8(const void *ptr)
91 + : reg((__m128i)_mm_loadu_si128((__m128i *)ptr)) {}
92 +
93 + explicit BF16Vec8(const FP32Vec8 &);
94 +
95 + void save(void *ptr) const { *reinterpret_cast<__m128i *>(ptr) = reg; }
96 +};
97 +
98 +struct BF16Vec16 : public Vec<BF16Vec16> {
99 + constexpr static int VEC_ELEM_NUM = 16;
100 +
101 + __m256i reg;
102 +
103 + explicit BF16Vec16(const void *ptr)
104 + : reg((__m256i)_mm256_loadu_si256((__m256i *)ptr)) {}
105 +
106 + explicit BF16Vec16(const FP32Vec16 &);
107 +
108 + void save(void *ptr) const { *reinterpret_cast<__m256i *>(ptr) = reg; }
109 +};
110 +
111 +#ifdef __AVX512F__
112 +struct BF16Vec32 : public Vec<BF16Vec32> {
113 + constexpr static int VEC_ELEM_NUM = 32;
114 +
115 + __m512i reg;
116 +
117 + explicit BF16Vec32(const void *ptr) : reg((__m512i)_mm512_loadu_si512(ptr)) {}
118 +
119 + explicit BF16Vec32(__m512i data) : reg(data) {}
120 +
121 + explicit BF16Vec32(BF16Vec8 &vec8_data)
122 + : reg((__m512i)_mm512_inserti32x4(
123 + _mm512_inserti32x4(_mm512_inserti32x4(_mm512_castsi128_si512(
124 + (__m128i)vec8_data.reg),
125 + (__m128i)vec8_data.reg, 1),
126 + (__m128i)vec8_data.reg, 2),
127 + (__m128i)vec8_data.reg, 3)) {}
128 +
129 + void save(void *ptr) const { *reinterpret_cast<__m512i *>(ptr) = reg; }
130 +};
131 +#else
132 +struct BF16Vec32 : public Vec<BF16Vec32> {
133 + constexpr static int VEC_ELEM_NUM = 32;
134 +
135 + __m256i reg_low;
136 + __m256i reg_high;
137 +
138 + explicit BF16Vec32(const void *ptr)
139 + : reg_low(_mm256_loadu_si256((__m256i const *)ptr)),
140 + reg_high(_mm256_loadu_si256((__m256i const *)ptr + 1)) {}
141 +
142 + explicit BF16Vec32(__m256i low, __m256i high) : reg_low(low),
143 + reg_high(high) {}
144 +
145 + explicit BF16Vec32(BF16Vec8 &vec8_data)
146 + : reg_low((__m256i)_mm256_inserti32x4(
147 + _mm256_castsi128_si256((__m128i)vec8_data.reg),
148 + (__m128i)vec8_data.reg, 1)),
149 + reg_high((__m256i)_mm256_inserti32x4(
150 + _mm256_castsi128_si256((__m128i)vec8_data.reg),
151 + (__m128i)vec8_data.reg, 1)) {}
152 +
153 + void save(void *ptr) const {
154 + *reinterpret_cast<__m256i *>(ptr) = reg_low;
155 + *reinterpret_cast<__m256i *>((__m256i *)ptr + 1) = reg_high;
156 + }
157 +};
158 +#endif
159 +
160 +struct FP32Vec4 : public Vec<FP32Vec4> {
161 + constexpr static int VEC_ELEM_NUM = 4;
162 + union AliasReg {
163 + __m128 reg;
164 + float values[VEC_ELEM_NUM];
165 + };
166 +
167 + __m128 reg;
168 +
169 + explicit FP32Vec4(float v) : reg(_mm_set1_ps(v)) {}
170 +
171 + explicit FP32Vec4() : reg(_mm_set1_ps(0.0)) {}
172 +
173 + explicit FP32Vec4(const float *ptr) : reg(_mm_loadu_ps(ptr)) {}
174 +
175 + explicit FP32Vec4(__m128 data) : reg(data) {}
176 +
177 + explicit FP32Vec4(const FP32Vec4 &data) : reg(data.reg) {}
178 +};
179 +
180 +struct FP32Vec8 : public Vec<FP32Vec8> {
181 + constexpr static int VEC_ELEM_NUM = 8;
182 + union AliasReg {
183 + __m256 reg;
184 + float values[VEC_ELEM_NUM];
185 + };
186 +
187 + __m256 reg;
188 +
189 + explicit FP32Vec8(float v) : reg(_mm256_set1_ps(v)) {}
190 +
191 + explicit FP32Vec8() : reg(_mm256_set1_ps(0.0)) {}
192 +
193 + explicit FP32Vec8(const float *ptr) : reg(_mm256_loadu_ps(ptr)) {}
194 +
195 + explicit FP32Vec8(__m256 data) : reg(data) {}
196 +
197 + explicit FP32Vec8(const FP32Vec8 &data) : reg(data.reg) {}
198 +
199 +#ifdef __AVX512FP16__
200 + explicit FP32Vec8(__m128h v) : reg(_mm256_cvtph_ps(_mm_castph_si128(v))) {}
201 +#endif
202 +
203 + explicit FP32Vec8(const BF16Vec8 &v)
204 + : reg(_mm256_castsi256_ps(
205 + _mm256_bslli_epi128(_mm256_cvtepu16_epi32(v.reg), 2))) {}
206 +
207 + float reduce_sum() const {
208 + AliasReg ar;
209 + ar.reg = reg;
210 + float result = 0;
211 + unroll_loop<int, VEC_ELEM_NUM>([&result, &ar](int i) { result += ar.values[i]; });
212 +
213 + return result;
214 + }
215 +
216 + FP32Vec8 exp() const {
217 + AliasReg ar;
218 + ar.reg = reg;
219 + return FP32Vec8(_mm256_set_ps(expf(ar.values[7]), expf(ar.values[6]),
220 + expf(ar.values[5]), expf(ar.values[4]),
221 + expf(ar.values[3]), expf(ar.values[2]),
222 + expf(ar.values[1]), expf(ar.values[0])));
223 + }
224 +
225 + FP32Vec8 tanh() const {
226 + AliasReg ar;
227 + ar.reg = reg;
228 + return FP32Vec8(_mm256_set_ps(tanhf(ar.values[7]), tanhf(ar.values[6]),
229 + tanhf(ar.values[5]), tanhf(ar.values[4]),
230 + tanhf(ar.values[3]), tanhf(ar.values[2]),
231 + tanhf(ar.values[1]), tanhf(ar.values[0])));
232 + }
233 +
234 + FP32Vec8 er() const {
235 + AliasReg ar;
236 + ar.reg = reg;
237 + return FP32Vec8(_mm256_set_ps(erf(ar.values[7]), erf(ar.values[6]),
238 + erf(ar.values[5]), erf(ar.values[4]),
239 + erf(ar.values[3]), erf(ar.values[2]),
240 + erf(ar.values[1]), erf(ar.values[0])));
241 + }
242 +
243 + FP32Vec8 operator*(const FP32Vec8 &b) const {
244 + return FP32Vec8(_mm256_mul_ps(reg, b.reg));
245 + }
246 +
247 + FP32Vec8 operator+(const FP32Vec8 &b) const {
248 + return FP32Vec8(_mm256_add_ps(reg, b.reg));
249 + }
250 +
251 + FP32Vec8 operator-(const FP32Vec8 &b) const {
252 + return FP32Vec8(_mm256_sub_ps(reg, b.reg));
253 + }
254 +
255 + FP32Vec8 operator/(const FP32Vec8 &b) const {
256 + return FP32Vec8(_mm256_div_ps(reg, b.reg));
257 + }
258 +
259 + void save(float *ptr) const { _mm256_storeu_ps(ptr, reg); }
260 +};
261 +
262 +#ifdef __AVX512F__
263 +struct FP32Vec16 : public Vec<FP32Vec16> {
264 + constexpr static int VEC_ELEM_NUM = 16;
265 + union AliasReg {
266 + __m512 reg;
267 + float values[VEC_ELEM_NUM];
268 + };
269 +
270 + __m512 reg;
271 +
272 + explicit FP32Vec16(float v) : reg(_mm512_set1_ps(v)) {}
273 +
274 + explicit FP32Vec16() : reg(_mm512_set1_ps(0.0)) {}
275 +
276 + explicit FP32Vec16(const float *ptr) : reg(_mm512_loadu_ps(ptr)) {}
277 +
278 + explicit FP32Vec16(__m512 data) : reg(data) {}
279 +
280 + explicit FP32Vec16(const FP32Vec16 &data) : reg(data.reg) {}
281 +
282 + explicit FP32Vec16(const FP32Vec4 &data)
283 + : reg((__m512)_mm512_inserti32x4(
284 + _mm512_inserti32x4(
285 + _mm512_inserti32x4(_mm512_castsi128_si512((__m128i)data.reg),
286 + (__m128i)data.reg, 1),
287 + (__m128i)data.reg, 2),
288 + (__m128i)data.reg, 3)) {}
289 +
290 + explicit FP32Vec16(const FP32Vec8 &data)
291 + : reg((__m512)_mm512_inserti32x8(
292 + _mm512_castsi256_si512((__m256i)data.reg), (__m256i)data.reg, 1)) {}
293 +
294 + explicit FP32Vec16(const BF16Vec16 &v)
295 + : reg(_mm512_castsi512_ps(
296 + _mm512_bslli_epi128(_mm512_cvtepu16_epi32(v.reg), 2))) {}
297 +
298 + explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
299 +
300 + FP32Vec16 operator*(const FP32Vec16 &b) const {
301 + return FP32Vec16(_mm512_mul_ps(reg, b.reg));
302 + }
303 +
304 + FP32Vec16 operator+(const FP32Vec16 &b) const {
305 + return FP32Vec16(_mm512_add_ps(reg, b.reg));
306 + }
307 +
308 + FP32Vec16 operator-(const FP32Vec16 &b) const {
309 + return FP32Vec16(_mm512_sub_ps(reg, b.reg));
310 + }
311 +
312 + FP32Vec16 operator/(const FP32Vec16 &b) const {
313 + return FP32Vec16(_mm512_div_ps(reg, b.reg));
314 + }
315 +
316 + float reduce_sum() const { return _mm512_reduce_add_ps(reg); }
317 +
318 + template <int group_size> float reduce_sub_sum(int idx) {
319 + static_assert(VEC_ELEM_NUM % group_size == 0);
320 + constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
321 + __mmask16 mask = _cvtu32_mask16(base_mask << (idx * group_size));
322 + return _mm512_mask_reduce_add_ps(mask, reg);
323 + }
324 +
325 + void save(float *ptr) const { _mm512_storeu_ps(ptr, reg); }
326 +};
327 +#else
328 +struct FP32Vec16 : public Vec<FP32Vec16> {
329 + constexpr static int VEC_ELEM_NUM = 16;
330 +
331 + union AliasReg {
332 + __m256 reg;
333 + float values[8];
334 + };
335 +
336 + __m256 reg_low;
337 + __m256 reg_high;
338 +
339 + explicit FP32Vec16(float v) : reg_low(_mm256_set1_ps(v)),
340 + reg_high(_mm256_set1_ps(v)) {}
341 +
342 + explicit FP32Vec16() : reg_low(_mm256_set1_ps(0.0)),
343 + reg_high(_mm256_set1_ps(0.0)) {}
344 +
345 + explicit FP32Vec16(const float *ptr) : reg_low(_mm256_loadu_ps(ptr)),
346 + reg_high(_mm256_loadu_ps(ptr + 8)) {}
347 +
348 + explicit FP32Vec16(__m256 low, __m256 high) : reg_low(low), reg_high(high) {}
349 +
350 + explicit FP32Vec16(const FP32Vec16 &data) : reg_low(data.reg_low),
351 + reg_high(data.reg_high) {}
352 +
353 + explicit FP32Vec16(const FP32Vec4 &data)
354 + : reg_low((__m256)_mm256_inserti128_si256(
355 + _mm256_castsi128_si256((__m128i)data.reg),
356 + (__m128i)data.reg, 1)),
357 + reg_high((__m256)_mm256_inserti128_si256(
358 + _mm256_castsi128_si256((__m128i)data.reg),
359 + (__m128i)data.reg, 1)) {}
360 +
361 + explicit FP32Vec16(const FP32Vec8 &data)
362 + : reg_low(data.reg), reg_high(data.reg) {}
363 +
364 + explicit FP32Vec16(const BF16Vec16 &v) {
365 + __m128i low = _mm256_extractf128_si256(v.reg, 0);
366 + __m128i high = _mm256_extractf128_si256(v.reg, 1);
367 +
368 + __m256i v_low_epi32 = _mm256_cvtepu16_epi32(low);
369 + __m256i v_high_epi32 = _mm256_cvtepu16_epi32(high);
370 +
371 + __m256i v_low_shifted = _mm256_bslli_epi128(v_low_epi32, 2);
372 + __m256i v_high_shifted = _mm256_bslli_epi128(v_high_epi32, 2);
373 +
374 + reg_low = _mm256_castsi256_ps(v_low_shifted);
375 + reg_high = _mm256_castsi256_ps(v_high_shifted);
376 + }
377 +
378 + explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
379 +
380 + FP32Vec16 operator*(const FP32Vec16 &b) const {
381 + return FP32Vec16(_mm256_mul_ps(reg_low, b.reg_low),
382 + _mm256_mul_ps(reg_high, b.reg_high));
383 + }
384 +
385 + FP32Vec16 operator+(const FP32Vec16 &b) const {
386 + return FP32Vec16(_mm256_add_ps(reg_low, b.reg_low),
387 + _mm256_add_ps(reg_high, b.reg_high));
388 + }
389 +
390 + FP32Vec16 operator-(const FP32Vec16 &b) const {
391 + return FP32Vec16(_mm256_sub_ps(reg_low, b.reg_low),
392 + _mm256_sub_ps(reg_high, b.reg_high));
393 + }
394 +
395 + FP32Vec16 operator/(const FP32Vec16 &b) const {
396 + return FP32Vec16(_mm256_div_ps(reg_low, b.reg_low),
397 + _mm256_div_ps(reg_high, b.reg_high));
398 + }
399 +
400 + float reduce_sum() const {
401 + FP32Vec8 low = FP32Vec8(reg_low);
402 + FP32Vec8 high = FP32Vec8(reg_high);
403 + return low.reduce_sum() + high.reduce_sum();
404 + }
405 +
406 + template <int group_size> float reduce_sub_sum(int idx) {
407 + float sum = 0.0;
408 + static_assert(VEC_ELEM_NUM % group_size == 0);
409 + constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
410 + uint32_t mask = base_mask << (idx * group_size);
411 +
412 + AliasReg ar;
413 +
414 + auto func = [&sum, &mask, &ar](int i) {
415 + int flag = mask & 0x1;
416 + mask = mask >> 1;
417 + if (flag != 0) sum += ar.values[i];
418 + };
419 +
420 + ar.reg = reg_low;
421 + unroll_loop<int, 8>(func);
422 +
423 + ar.reg = reg_high;
424 + unroll_loop<int, 8>(func);
425 +
426 + return sum;
427 + }
428 +
429 + void save(float *ptr) const {
430 + _mm256_storeu_ps(ptr, reg_low);
431 + _mm256_storeu_ps(ptr + 8, reg_high);
432 + }
433 +};
434 +#endif
435 +
436 +template <typename T> struct VecType { using vec_type = void; };
437 +
438 +template <typename T> using vec_t = typename VecType<T>::vec_type;
439 +
440 +template <> struct VecType<float> { using vec_type = FP32Vec8; };
441 +
442 +#ifdef __AVX512FP16__
443 +template <> struct VecType<c10::Half> { using vec_type = FP16Vec16; };
444 +#endif
445 +
446 +template <> struct VecType<c10::BFloat16> { using vec_type = BF16Vec8; };
447 +
448 +template <typename T> void storeFP32(float v, T *ptr) { *ptr = v; }
449 +
450 +#ifdef __AVX512FP16__
451 +template <> inline void storeFP32<c10::Half>(float v, c10::Half *ptr) {
452 + *reinterpret_cast<_Float16 *>(ptr) = v;
453 +}
454 +#endif
455 +
456 +inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
457 + acc = acc + a * b;
458 +}
459 +
460 +#ifdef __AVX512BF16__
461 +template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
462 + *reinterpret_cast<__bfloat16 *>(ptr) = _mm_cvtness_sbh(v);
463 +}
464 +
465 +inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
466 + : reg((__m128i)_mm256_cvtneps_pbh(v.reg)) {}
467 +
468 +inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
469 + : reg((__m256i)_mm512_cvtneps_pbh(v.reg)) {}
470 +
471 +inline void fma(FP32Vec16 &acc, BF16Vec32 &a, BF16Vec32 &b) {
472 + acc.reg = _mm512_dpbf16_ps(acc.reg, (__m512bh)a.reg, (__m512bh)b.reg);
473 +}
474 +#else
475 +template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
476 + c10::BFloat16 __attribute__((__may_alias__)) *v_ptr =
477 + reinterpret_cast<c10::BFloat16 *>(&v);
478 + *ptr = *(v_ptr + 1);
479 +}
480 +
481 +#ifdef __AVX512F__
482 +inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
483 + : reg(_mm256_cvtepi32_epi16(
484 + _mm256_bsrli_epi128(_mm256_castps_si256(v.reg), 2))) {}
485 +
486 +inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
487 + : reg(_mm512_cvtepi32_epi16(
488 + _mm512_bsrli_epi128(_mm512_castps_si512(v.reg), 2))) {}
489 +#else
490 +namespace{
491 +__m128i FP32Vec8_to_BF16Vec8_avx2(__m256 a) {
492 + __m256i ai = _mm256_castps_si256(a);
493 + ai = _mm256_srli_epi32(ai, 16);
494 + ai = _mm256_packus_epi32(ai, ai);
495 + ai = _mm256_permute4x64_epi64(ai, 0b00111001);
496 + return _mm256_extracti128_si256(ai, 0);
497 +}
498 +}
499 +
500 +inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
501 + : reg(FP32Vec8_to_BF16Vec8_avx2(v.reg)) {}
502 +
503 +inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
504 + BF16Vec8 low = BF16Vec8(FP32Vec8(v.reg_low));
505 + BF16Vec8 high = BF16Vec8(FP32Vec8(v.reg_high));
506 + reg = _mm256_insertf128_si256(_mm256_castsi128_si256(low.reg), high.reg, 1);
507 +}
508 +#endif // __AVX512F__
509 +#endif // __AVX512BF16__
510 +
511 +inline void prefetch(const void *addr) { _mm_prefetch(addr, _MM_HINT_T1); }
512 +
513 +}; // namespace vec_op
514 +
515 +#endif
csrc/ops.h
+1
@@ -1,5 +1,6 @@
1 #pragma once
2
3 +#include <optional>
4 #include <torch/library.h>
5
6 void paged_attention_v1(
requirements-cpu.txt
+3 -3
@@ -2,6 +2,6 @@
2 -r requirements-common.txt
3
4 # Dependencies for x86_64 CPUs
5 -torch == 2.3.1+cpu
6 -torchvision == 0.18.1+cpu # required for the image processor of phi3v, this must be updated alongside torch
7 -triton >= 2.2.0 # FIXME(woosuk): This is a hack to avoid import error.
\ No newline at end of file
5 +torch == 2.3.1+cpu; platform_machine != "ppc64le"
6 +torchvision == 0.18.1+cpu; platform_machine != "ppc64le" # required for the image processor of phi3v, this must be updated alongside torch
7 +triton >= 2.2.0 # FIXME(woosuk): This is a hack to avoid import error.