A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
Support CPU inference with VSX PowerPC ISA (#5652)
Chip Kerchner committed
Jun 26, 2024 at 17:53 UTC
38a1674abbba38344543170cb552e88e7f619167
7 files changed
+1049
-511
Dockerfile.ppc64le
new
+22
@@ -0,0 +1,22 @@
1
+FROM mambaorg/micromamba
2
+ARG MAMBA_DOCKERFILE_ACTIVATE=1
3
+USER root
4
+
5
+RUN apt-get update -y && apt-get install -y git wget vim numactl gcc-12 g++-12 protobuf-compiler libprotobuf-dev && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
6
+
7
+# Some packages in requirements-cpu are installed here
8
+# IBM provides optimized packages for ppc64le processors in the open-ce project for mamba
9
+# Currently these may not be available for venv or pip directly
10
+RUN micromamba install -y -n base -c https://ftp.osuosl.org/pub/open-ce/1.11.0-p10/ -c defaults python=3.10 pytorch-cpu=2.1.2 torchvision-cpu=0.16.2 && micromamba clean --all --yes
11
+
12
+COPY ./ /workspace/vllm
13
+
14
+WORKDIR /workspace/vllm
15
+
16
+# These packages will be in rocketce eventually
17
+RUN pip install -v -r requirements-cpu.txt --prefer-binary --extra-index-url https://repo.fury.io/mgiessing
18
+
19
+RUN VLLM_TARGET_DEVICE=cpu python3 setup.py install
20
+
21
+WORKDIR /vllm-workspace
22
+ENTRYPOINT ["/opt/conda/bin/python3", "-m", "vllm.entrypoints.openai.api_server"]
cmake/cpu_extension.cmake
+10
-1
@@ -46,6 +46,8 @@ is_avx512_disabled(AVX512_DISABLED)
46
47
find_isa(${CPUINFO} "avx2" AVX2_FOUND)
48
find_isa(${CPUINFO} "avx512f" AVX512_FOUND)
49
+find_isa(${CPUINFO} "POWER10" POWER10_FOUND)
50
+find_isa(${CPUINFO} "POWER9" POWER9_FOUND)
51
52
if (AVX512_FOUND AND NOT AVX512_DISABLED)
53
list(APPEND CXX_COMPILE_FLAGS
@@ -68,8 +70,15 @@ if (AVX512_FOUND AND NOT AVX512_DISABLED)
70
elseif (AVX2_FOUND)
71
list(APPEND CXX_COMPILE_FLAGS "-mavx2")
72
message(WARNING "vLLM CPU backend using AVX2 ISA")
73
+elseif (POWER9_FOUND OR POWER10_FOUND)
74
+ message(STATUS "PowerPC detected")
75
+ # Check for PowerPC VSX support
76
+ list(APPEND CXX_COMPILE_FLAGS
77
+ "-mvsx"
78
+ "-mcpu=native"
79
+ "-mtune=native")
80
else()
72
- message(FATAL_ERROR "vLLM CPU backend requires AVX512 or AVX2 ISA support.")
81
+ message(FATAL_ERROR "vLLM CPU backend requires AVX512 or AVX2 or Power9+ ISA support.")
82
endif()
83
84
message(STATUS "CPU extension compile flags: ${CXX_COMPILE_FLAGS}")
csrc/cpu/cpu_types.hpp
+7
-507
@@ -2,514 +2,14 @@
2
#ifndef CPU_TYPES_HPP
3
#define CPU_TYPES_HPP
4
5
-#include <immintrin.h>
6
-#include <torch/all.h>
7
-
8
-#ifndef __AVX2__
9
-static_assert(false, "AVX2 must be supported for the current implementation.");
10
-#endif
11
-
12
-namespace vec_op {
13
-
14
-// FIXME: FP16 is not fully supported in Torch-CPU
15
-#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
16
- AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
17
- AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
18
-
19
-#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
20
- AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
21
-
22
-#ifndef CPU_OP_GUARD
23
-#define CPU_KERNEL_GUARD_IN(NAME)
24
-#define CPU_KERNEL_GUARD_OUT(NAME)
25
-#else
26
-#define CPU_KERNEL_GUARD_IN(NAME) \
27
- std::cout << #NAME << " invoked." << std::endl;
28
-#define CPU_KERNEL_GUARD_OUT(NAME) std::cout << #NAME << " exit." << std::endl;
29
-#endif
30
-
31
-#define FORCE_INLINE __attribute__((always_inline)) inline
32
-
33
-namespace {
34
-template <typename T, T... indexes, typename F>
35
-constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F &&f) {
36
- (f(std::integral_constant<T, indexes>{}), ...);
37
-}
38
-}; // namespace
39
-
40
-template <typename T, T count, typename F,
41
- typename = std::enable_if_t<std::is_invocable_v<F, T>>>
42
-constexpr void unroll_loop(F &&f) {
43
- unroll_loop_item(std::make_integer_sequence<T, count>{}, std::forward<F>(f));
44
-}
45
-
46
-template <typename T> struct Vec {
47
- constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; }
48
-};
49
-
50
-struct FP32Vec8;
51
-struct FP32Vec16;
52
-
53
-#ifdef __AVX512FP16__
54
-struct FP16Vec8 : public Vec<FP16Vec8> {
55
- constexpr static int VEC_ELEM_NUM = 8;
56
-
57
- __m128h reg;
58
-
59
- explicit FP16Vec8(_Float16 v) : reg(_mm_set1_ph(v)) {}
60
-
61
- explicit FP16Vec8(const void *ptr) : reg(_mm_loadu_ph(ptr)) {}
62
-
63
- explicit FP16Vec8(__m128h data) : reg(data) {}
64
-
65
- FP16Vec8 operator*(const FP16Vec8 &b) const {
66
- return FP16Vec8(_mm_mul_ph(reg, b.reg));
67
- }
68
-
69
- FP16Vec8 operator+(const FP16Vec8 &b) const {
70
- return FP16Vec8(_mm_add_ph(reg, b.reg));
71
- }
72
-
73
- FP16Vec8 operator-(const FP16Vec8 &b) const {
74
- return FP16Vec8(_mm_sub_ph(reg, b.reg));
75
- }
76
-
77
- FP16Vec8 operator/(const FP16Vec8 &b) const {
78
- return FP16Vec8(_mm_div_ph(reg, b.reg));
79
- }
80
-
81
- void save(void *ptr) const { _mm_storeu_ph(ptr, reg); }
82
-};
83
-#endif
84
-
85
-struct BF16Vec8 : public Vec<BF16Vec8> {
86
- constexpr static int VEC_ELEM_NUM = 8;
87
-
88
- __m128i reg;
89
-
90
- explicit BF16Vec8(const void *ptr)
91
- : reg((__m128i)_mm_loadu_si128((__m128i *)ptr)) {}
92
-
93
- explicit BF16Vec8(const FP32Vec8 &);
94
-
95
- void save(void *ptr) const { *reinterpret_cast<__m128i *>(ptr) = reg; }
96
-};
97
-
98
-struct BF16Vec16 : public Vec<BF16Vec16> {
99
- constexpr static int VEC_ELEM_NUM = 16;
100
-
101
- __m256i reg;
102
-
103
- explicit BF16Vec16(const void *ptr)
104
- : reg((__m256i)_mm256_loadu_si256((__m256i *)ptr)) {}
105
-
106
- explicit BF16Vec16(const FP32Vec16 &);
107
-
108
- void save(void *ptr) const { *reinterpret_cast<__m256i *>(ptr) = reg; }
109
-};
110
-
111
-#ifdef __AVX512F__
112
-struct BF16Vec32 : public Vec<BF16Vec32> {
113
- constexpr static int VEC_ELEM_NUM = 32;
114
-
115
- __m512i reg;
116
-
117
- explicit BF16Vec32(const void *ptr) : reg((__m512i)_mm512_loadu_si512(ptr)) {}
118
-
119
- explicit BF16Vec32(__m512i data) : reg(data) {}
120
-
121
- explicit BF16Vec32(BF16Vec8 &vec8_data)
122
- : reg((__m512i)_mm512_inserti32x4(
123
- _mm512_inserti32x4(_mm512_inserti32x4(_mm512_castsi128_si512(
124
- (__m128i)vec8_data.reg),
125
- (__m128i)vec8_data.reg, 1),
126
- (__m128i)vec8_data.reg, 2),
127
- (__m128i)vec8_data.reg, 3)) {}
128
-
129
- void save(void *ptr) const { *reinterpret_cast<__m512i *>(ptr) = reg; }
130
-};
131
-#else
132
-struct BF16Vec32 : public Vec<BF16Vec32> {
133
- constexpr static int VEC_ELEM_NUM = 32;
134
-
135
- __m256i reg_low;
136
- __m256i reg_high;
137
-
138
- explicit BF16Vec32(const void *ptr)
139
- : reg_low(_mm256_loadu_si256((__m256i const *)ptr)),
140
- reg_high(_mm256_loadu_si256((__m256i const *)ptr + 1)) {}
141
-
142
- explicit BF16Vec32(__m256i low, __m256i high) : reg_low(low),
143
- reg_high(high) {}
144
-
145
- explicit BF16Vec32(BF16Vec8 &vec8_data)
146
- : reg_low((__m256i)_mm256_inserti32x4(
147
- _mm256_castsi128_si256((__m128i)vec8_data.reg),
148
- (__m128i)vec8_data.reg, 1)),
149
- reg_high((__m256i)_mm256_inserti32x4(
150
- _mm256_castsi128_si256((__m128i)vec8_data.reg),
151
- (__m128i)vec8_data.reg, 1)) {}
152
-
153
- void save(void *ptr) const {
154
- *reinterpret_cast<__m256i *>(ptr) = reg_low;
155
- *reinterpret_cast<__m256i *>((__m256i *)ptr + 1) = reg_high;
156
- }
157
-};
158
-#endif
159
-
160
-struct FP32Vec4 : public Vec<FP32Vec4> {
161
- constexpr static int VEC_ELEM_NUM = 4;
162
- union AliasReg {
163
- __m128 reg;
164
- float values[VEC_ELEM_NUM];
165
- };
166
-
167
- __m128 reg;
168
-
169
- explicit FP32Vec4(float v) : reg(_mm_set1_ps(v)) {}
170
-
171
- explicit FP32Vec4() : reg(_mm_set1_ps(0.0)) {}
172
-
173
- explicit FP32Vec4(const float *ptr) : reg(_mm_loadu_ps(ptr)) {}
174
-
175
- explicit FP32Vec4(__m128 data) : reg(data) {}
176
-
177
- explicit FP32Vec4(const FP32Vec4 &data) : reg(data.reg) {}
178
-};
179
-
180
-struct FP32Vec8 : public Vec<FP32Vec8> {
181
- constexpr static int VEC_ELEM_NUM = 8;
182
- union AliasReg {
183
- __m256 reg;
184
- float values[VEC_ELEM_NUM];
185
- };
186
-
187
- __m256 reg;
188
-
189
- explicit FP32Vec8(float v) : reg(_mm256_set1_ps(v)) {}
190
-
191
- explicit FP32Vec8() : reg(_mm256_set1_ps(0.0)) {}
192
-
193
- explicit FP32Vec8(const float *ptr) : reg(_mm256_loadu_ps(ptr)) {}
194
-
195
- explicit FP32Vec8(__m256 data) : reg(data) {}
196
-
197
- explicit FP32Vec8(const FP32Vec8 &data) : reg(data.reg) {}
198
-
199
-#ifdef __AVX512FP16__
200
- explicit FP32Vec8(__m128h v) : reg(_mm256_cvtph_ps(_mm_castph_si128(v))) {}
201
-#endif
202
-
203
- explicit FP32Vec8(const BF16Vec8 &v)
204
- : reg(_mm256_castsi256_ps(
205
- _mm256_bslli_epi128(_mm256_cvtepu16_epi32(v.reg), 2))) {}
206
-
207
- float reduce_sum() const {
208
- AliasReg ar;
209
- ar.reg = reg;
210
- float result = 0;
211
- unroll_loop<int, VEC_ELEM_NUM>([&result, &ar](int i) { result += ar.values[i]; });
212
-
213
- return result;
214
- }
215
-
216
- FP32Vec8 exp() const {
217
- AliasReg ar;
218
- ar.reg = reg;
219
- return FP32Vec8(_mm256_set_ps(expf(ar.values[7]), expf(ar.values[6]),
220
- expf(ar.values[5]), expf(ar.values[4]),
221
- expf(ar.values[3]), expf(ar.values[2]),
222
- expf(ar.values[1]), expf(ar.values[0])));
223
- }
224
-
225
- FP32Vec8 tanh() const {
226
- AliasReg ar;
227
- ar.reg = reg;
228
- return FP32Vec8(_mm256_set_ps(tanhf(ar.values[7]), tanhf(ar.values[6]),
229
- tanhf(ar.values[5]), tanhf(ar.values[4]),
230
- tanhf(ar.values[3]), tanhf(ar.values[2]),
231
- tanhf(ar.values[1]), tanhf(ar.values[0])));
232
- }
233
-
234
- FP32Vec8 er() const {
235
- AliasReg ar;
236
- ar.reg = reg;
237
- return FP32Vec8(_mm256_set_ps(erf(ar.values[7]), erf(ar.values[6]),
238
- erf(ar.values[5]), erf(ar.values[4]),
239
- erf(ar.values[3]), erf(ar.values[2]),
240
- erf(ar.values[1]), erf(ar.values[0])));
241
- }
242
-
243
- FP32Vec8 operator*(const FP32Vec8 &b) const {
244
- return FP32Vec8(_mm256_mul_ps(reg, b.reg));
245
- }
246
-
247
- FP32Vec8 operator+(const FP32Vec8 &b) const {
248
- return FP32Vec8(_mm256_add_ps(reg, b.reg));
249
- }
250
-
251
- FP32Vec8 operator-(const FP32Vec8 &b) const {
252
- return FP32Vec8(_mm256_sub_ps(reg, b.reg));
253
- }
254
-
255
- FP32Vec8 operator/(const FP32Vec8 &b) const {
256
- return FP32Vec8(_mm256_div_ps(reg, b.reg));
257
- }
258
-
259
- void save(float *ptr) const { _mm256_storeu_ps(ptr, reg); }
260
-};
261
-
262
-#ifdef __AVX512F__
263
-struct FP32Vec16 : public Vec<FP32Vec16> {
264
- constexpr static int VEC_ELEM_NUM = 16;
265
- union AliasReg {
266
- __m512 reg;
267
- float values[VEC_ELEM_NUM];
268
- };
269
-
270
- __m512 reg;
271
-
272
- explicit FP32Vec16(float v) : reg(_mm512_set1_ps(v)) {}
273
-
274
- explicit FP32Vec16() : reg(_mm512_set1_ps(0.0)) {}
275
-
276
- explicit FP32Vec16(const float *ptr) : reg(_mm512_loadu_ps(ptr)) {}
277
-
278
- explicit FP32Vec16(__m512 data) : reg(data) {}
279
-
280
- explicit FP32Vec16(const FP32Vec16 &data) : reg(data.reg) {}
281
-
282
- explicit FP32Vec16(const FP32Vec4 &data)
283
- : reg((__m512)_mm512_inserti32x4(
284
- _mm512_inserti32x4(
285
- _mm512_inserti32x4(_mm512_castsi128_si512((__m128i)data.reg),
286
- (__m128i)data.reg, 1),
287
- (__m128i)data.reg, 2),
288
- (__m128i)data.reg, 3)) {}
289
-
290
- explicit FP32Vec16(const FP32Vec8 &data)
291
- : reg((__m512)_mm512_inserti32x8(
292
- _mm512_castsi256_si512((__m256i)data.reg), (__m256i)data.reg, 1)) {}
293
-
294
- explicit FP32Vec16(const BF16Vec16 &v)
295
- : reg(_mm512_castsi512_ps(
296
- _mm512_bslli_epi128(_mm512_cvtepu16_epi32(v.reg), 2))) {}
297
-
298
- explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
299
-
300
- FP32Vec16 operator*(const FP32Vec16 &b) const {
301
- return FP32Vec16(_mm512_mul_ps(reg, b.reg));
302
- }
303
-
304
- FP32Vec16 operator+(const FP32Vec16 &b) const {
305
- return FP32Vec16(_mm512_add_ps(reg, b.reg));
306
- }
307
-
308
- FP32Vec16 operator-(const FP32Vec16 &b) const {
309
- return FP32Vec16(_mm512_sub_ps(reg, b.reg));
310
- }
311
-
312
- FP32Vec16 operator/(const FP32Vec16 &b) const {
313
- return FP32Vec16(_mm512_div_ps(reg, b.reg));
314
- }
315
-
316
- float reduce_sum() const { return _mm512_reduce_add_ps(reg); }
317
-
318
- template <int group_size> float reduce_sub_sum(int idx) {
319
- static_assert(VEC_ELEM_NUM % group_size == 0);
320
- constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
321
- __mmask16 mask = _cvtu32_mask16(base_mask << (idx * group_size));
322
- return _mm512_mask_reduce_add_ps(mask, reg);
323
- }
324
-
325
- void save(float *ptr) const { _mm512_storeu_ps(ptr, reg); }
326
-};
5
+#if defined(__x86_64__)
6
+ //x86 implementation
7
+ #include "cpu_types_x86.hpp"
8
+#elif defined(__POWER9_VECTOR__)
9
+ //ppc implementation
10
+ #include "cpu_types_vsx.hpp"
11
#else
328
-struct FP32Vec16 : public Vec<FP32Vec16> {
329
- constexpr static int VEC_ELEM_NUM = 16;
330
-
331
- union AliasReg {
332
- __m256 reg;
333
- float values[8];
334
- };
335
-
336
- __m256 reg_low;
337
- __m256 reg_high;
338
-
339
- explicit FP32Vec16(float v) : reg_low(_mm256_set1_ps(v)),
340
- reg_high(_mm256_set1_ps(v)) {}
341
-
342
- explicit FP32Vec16() : reg_low(_mm256_set1_ps(0.0)),
343
- reg_high(_mm256_set1_ps(0.0)) {}
344
-
345
- explicit FP32Vec16(const float *ptr) : reg_low(_mm256_loadu_ps(ptr)),
346
- reg_high(_mm256_loadu_ps(ptr + 8)) {}
347
-
348
- explicit FP32Vec16(__m256 low, __m256 high) : reg_low(low), reg_high(high) {}
349
-
350
- explicit FP32Vec16(const FP32Vec16 &data) : reg_low(data.reg_low),
351
- reg_high(data.reg_high) {}
352
-
353
- explicit FP32Vec16(const FP32Vec4 &data)
354
- : reg_low((__m256)_mm256_inserti128_si256(
355
- _mm256_castsi128_si256((__m128i)data.reg),
356
- (__m128i)data.reg, 1)),
357
- reg_high((__m256)_mm256_inserti128_si256(
358
- _mm256_castsi128_si256((__m128i)data.reg),
359
- (__m128i)data.reg, 1)) {}
360
-
361
- explicit FP32Vec16(const FP32Vec8 &data)
362
- : reg_low(data.reg), reg_high(data.reg) {}
363
-
364
- explicit FP32Vec16(const BF16Vec16 &v) {
365
- __m128i low = _mm256_extractf128_si256(v.reg, 0);
366
- __m128i high = _mm256_extractf128_si256(v.reg, 1);
367
-
368
- __m256i v_low_epi32 = _mm256_cvtepu16_epi32(low);
369
- __m256i v_high_epi32 = _mm256_cvtepu16_epi32(high);
370
-
371
- __m256i v_low_shifted = _mm256_bslli_epi128(v_low_epi32, 2);
372
- __m256i v_high_shifted = _mm256_bslli_epi128(v_high_epi32, 2);
373
-
374
- reg_low = _mm256_castsi256_ps(v_low_shifted);
375
- reg_high = _mm256_castsi256_ps(v_high_shifted);
376
- }
377
-
378
- explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
379
-
380
- FP32Vec16 operator*(const FP32Vec16 &b) const {
381
- return FP32Vec16(_mm256_mul_ps(reg_low, b.reg_low),
382
- _mm256_mul_ps(reg_high, b.reg_high));
383
- }
384
-
385
- FP32Vec16 operator+(const FP32Vec16 &b) const {
386
- return FP32Vec16(_mm256_add_ps(reg_low, b.reg_low),
387
- _mm256_add_ps(reg_high, b.reg_high));
388
- }
389
-
390
- FP32Vec16 operator-(const FP32Vec16 &b) const {
391
- return FP32Vec16(_mm256_sub_ps(reg_low, b.reg_low),
392
- _mm256_sub_ps(reg_high, b.reg_high));
393
- }
394
-
395
- FP32Vec16 operator/(const FP32Vec16 &b) const {
396
- return FP32Vec16(_mm256_div_ps(reg_low, b.reg_low),
397
- _mm256_div_ps(reg_high, b.reg_high));
398
- }
399
-
400
- float reduce_sum() const {
401
- FP32Vec8 low = FP32Vec8(reg_low);
402
- FP32Vec8 high = FP32Vec8(reg_high);
403
- return low.reduce_sum() + high.reduce_sum();
404
- }
405
-
406
- template <int group_size> float reduce_sub_sum(int idx) {
407
- float sum = 0.0;
408
- static_assert(VEC_ELEM_NUM % group_size == 0);
409
- constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
410
- uint32_t mask = base_mask << (idx * group_size);
411
-
412
- AliasReg ar;
413
-
414
- auto func = [&sum, &mask, &ar](int i) {
415
- int flag = mask & 0x1;
416
- mask = mask >> 1;
417
- if (flag != 0) sum += ar.values[i];
418
- };
419
-
420
- ar.reg = reg_low;
421
- unroll_loop<int, 8>(func);
422
-
423
- ar.reg = reg_high;
424
- unroll_loop<int, 8>(func);
425
-
426
- return sum;
427
- }
428
-
429
- void save(float *ptr) const {
430
- _mm256_storeu_ps(ptr, reg_low);
431
- _mm256_storeu_ps(ptr + 8, reg_high);
432
- }
433
-};
434
-#endif
435
-
436
-template <typename T> struct VecType { using vec_type = void; };
437
-
438
-template <typename T> using vec_t = typename VecType<T>::vec_type;
439
-
440
-template <> struct VecType<float> { using vec_type = FP32Vec8; };
441
-
442
-#ifdef __AVX512FP16__
443
-template <> struct VecType<c10::Half> { using vec_type = FP16Vec16; };
12
+ #warning "unsupported vLLM cpu implementation"
13
#endif
14
446
-template <> struct VecType<c10::BFloat16> { using vec_type = BF16Vec8; };
447
-
448
-template <typename T> void storeFP32(float v, T *ptr) { *ptr = v; }
449
-
450
-#ifdef __AVX512FP16__
451
-template <> inline void storeFP32<c10::Half>(float v, c10::Half *ptr) {
452
- *reinterpret_cast<_Float16 *>(ptr) = v;
453
-}
454
-#endif
455
-
456
-inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
457
- acc = acc + a * b;
458
-}
459
-
460
-#ifdef __AVX512BF16__
461
-template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
462
- *reinterpret_cast<__bfloat16 *>(ptr) = _mm_cvtness_sbh(v);
463
-}
464
-
465
-inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
466
- : reg((__m128i)_mm256_cvtneps_pbh(v.reg)) {}
467
-
468
-inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
469
- : reg((__m256i)_mm512_cvtneps_pbh(v.reg)) {}
470
-
471
-inline void fma(FP32Vec16 &acc, BF16Vec32 &a, BF16Vec32 &b) {
472
- acc.reg = _mm512_dpbf16_ps(acc.reg, (__m512bh)a.reg, (__m512bh)b.reg);
473
-}
474
-#else
475
-template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
476
- c10::BFloat16 __attribute__((__may_alias__)) *v_ptr =
477
- reinterpret_cast<c10::BFloat16 *>(&v);
478
- *ptr = *(v_ptr + 1);
479
-}
480
-
481
-#ifdef __AVX512F__
482
-inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
483
- : reg(_mm256_cvtepi32_epi16(
484
- _mm256_bsrli_epi128(_mm256_castps_si256(v.reg), 2))) {}
485
-
486
-inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
487
- : reg(_mm512_cvtepi32_epi16(
488
- _mm512_bsrli_epi128(_mm512_castps_si512(v.reg), 2))) {}
489
-#else
490
-namespace{
491
-__m128i FP32Vec8_to_BF16Vec8_avx2(__m256 a) {
492
- __m256i ai = _mm256_castps_si256(a);
493
- ai = _mm256_srli_epi32(ai, 16);
494
- ai = _mm256_packus_epi32(ai, ai);
495
- ai = _mm256_permute4x64_epi64(ai, 0b00111001);
496
- return _mm256_extracti128_si256(ai, 0);
497
-}
498
-}
499
-
500
-inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
501
- : reg(FP32Vec8_to_BF16Vec8_avx2(v.reg)) {}
502
-
503
-inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
504
- BF16Vec8 low = BF16Vec8(FP32Vec8(v.reg_low));
505
- BF16Vec8 high = BF16Vec8(FP32Vec8(v.reg_high));
506
- reg = _mm256_insertf128_si256(_mm256_castsi128_si256(low.reg), high.reg, 1);
507
-}
508
-#endif // __AVX512F__
509
-#endif // __AVX512BF16__
510
-
511
-inline void prefetch(const void *addr) { _mm_prefetch(addr, _MM_HINT_T1); }
512
-
513
-}; // namespace vec_op
514
-
15
#endif
csrc/cpu/cpu_types_vsx.hpp
new
+491
@@ -0,0 +1,491 @@
1
+
2
+#ifndef CPU_TYPES_VSX_HPP
3
+#define CPU_TYPES_VSX_HPP
4
+
5
+#include <altivec.h>
6
+#include <cmath>
7
+#include <torch/all.h>
8
+
9
+namespace vec_op {
10
+
11
+// FIXME: FP16 is not fully supported in Torch-CPU
12
+#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
13
+ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
14
+ AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
15
+
16
+#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
17
+ AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
18
+
19
+#ifndef CPU_OP_GUARD
20
+#define CPU_KERNEL_GUARD_IN(NAME)
21
+#define CPU_KERNEL_GUARD_OUT(NAME)
22
+#else
23
+#define CPU_KERNEL_GUARD_IN(NAME) \
24
+ std::cout << #NAME << " invoked." << std::endl;
25
+#define CPU_KERNEL_GUARD_OUT(NAME) std::cout << #NAME << " exit." << std::endl;
26
+#endif
27
+
28
+#define FORCE_INLINE __attribute__((always_inline)) inline
29
+
30
+namespace {
31
+template <typename T, T... indexes, typename F>
32
+constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F &&f) {
33
+ (f(std::integral_constant<T, indexes>{}), ...);
34
+}
35
+}; // namespace
36
+
37
+template <typename T, T count, typename F,
38
+ typename = std::enable_if_t<std::is_invocable_v<F, T>>>
39
+constexpr void unroll_loop(F &&f) {
40
+ unroll_loop_item(std::make_integer_sequence<T, count>{}, std::forward<F>(f));
41
+}
42
+
43
+template <typename T> struct Vec {
44
+ constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; }
45
+};
46
+
47
+typedef struct ss16x8x2_t {
48
+ __vector signed short val[2];
49
+} ss16x8x2_t;
50
+
51
+typedef struct ss16x8x4_t {
52
+ __vector signed short val[4];
53
+} ss16x8x4_t;
54
+
55
+typedef struct f32x4x2_t {
56
+ __vector float val[2];
57
+} f32x4x2_t;
58
+
59
+typedef struct f32x4x4_t {
60
+ __vector float val[4];
61
+} f32x4x4_t;
62
+
63
+struct FP32Vec8;
64
+struct FP32Vec16;
65
+
66
+struct BF16Vec8 : public Vec<BF16Vec8> {
67
+ constexpr static int VEC_ELEM_NUM = 8;
68
+
69
+ __vector signed short reg;
70
+
71
+ explicit BF16Vec8(const void *ptr)
72
+ : reg((__vector signed short)vec_xl(0, (__vector signed short *)ptr)) {}
73
+
74
+ explicit BF16Vec8(const FP32Vec8 &);
75
+
76
+ void save(void *ptr) const { *reinterpret_cast<__vector signed short *>(ptr) = reg; }
77
+};
78
+
79
+struct BF16Vec16 : public Vec<BF16Vec16> {
80
+ constexpr static int VEC_ELEM_NUM = 16;
81
+
82
+ ss16x8x2_t reg;
83
+
84
+ explicit BF16Vec16(const void *ptr) {
85
+ // Load 256 bits in two parts
86
+ reg.val[0] = (__vector signed short)vec_xl(0, (signed short *)ptr);
87
+ reg.val[1] = (__vector signed short)vec_xl(16, (signed short *)ptr);
88
+ }
89
+
90
+ explicit BF16Vec16(const FP32Vec16 &);
91
+
92
+ void save(void *ptr) const {
93
+ // Save 256 bits in two parts
94
+ vec_xst(reg.val[0], 0, (signed short *)ptr);
95
+ vec_xst(reg.val[1], 16, (signed short *)ptr);
96
+ }
97
+};
98
+
99
+const static __vector signed short zero = vec_splats((signed short)0);
100
+
101
+struct BF16Vec32 : public Vec<BF16Vec32> {
102
+ constexpr static int VEC_ELEM_NUM = 32;
103
+
104
+ ss16x8x4_t reg;
105
+ explicit BF16Vec32(const void *ptr)
106
+ : reg(*reinterpret_cast<const ss16x8x4_t *>(ptr)) {}
107
+
108
+ explicit BF16Vec32(ss16x8x4_t data) : reg(data) {}
109
+
110
+ explicit BF16Vec32(const BF16Vec8 &vec8_data) : reg({
111
+ vec8_data.reg,
112
+ vec8_data.reg,
113
+ vec8_data.reg,
114
+ vec8_data.reg
115
+ }) {}
116
+
117
+ void save(void *ptr) const { *reinterpret_cast<ss16x8x4_t *>(ptr) = reg; }
118
+};
119
+
120
+struct FP32Vec4 : public Vec<FP32Vec4> {
121
+ constexpr static int VEC_ELEM_NUM = 4;
122
+ union AliasReg {
123
+ __vector float reg;
124
+ float values[VEC_ELEM_NUM];
125
+ };
126
+
127
+ __vector float reg;
128
+
129
+ explicit FP32Vec4(float v) : reg(vec_splats(v)) {}
130
+
131
+ explicit FP32Vec4() : reg(vec_splats(0.0f)) {}
132
+
133
+ explicit FP32Vec4(const float *ptr) : reg(vec_xl(0, ptr)) {}
134
+
135
+ explicit FP32Vec4(__vector float data) : reg(data) {}
136
+
137
+ explicit FP32Vec4(const FP32Vec4 &data) : reg(data.reg) {}
138
+};
139
+
140
+struct FP32Vec8 : public Vec<FP32Vec8> {
141
+ constexpr static int VEC_ELEM_NUM = 8;
142
+ union AliasReg {
143
+ f32x4x2_t reg;
144
+ float values[VEC_ELEM_NUM];
145
+ };
146
+
147
+ f32x4x2_t reg;
148
+
149
+ explicit FP32Vec8(float v) {
150
+ reg.val[0] = vec_splats(v);
151
+ reg.val[1] = vec_splats(v);
152
+ }
153
+
154
+ explicit FP32Vec8() {
155
+ reg.val[0] = vec_splats(0.0f);
156
+ reg.val[1] = vec_splats(0.0f);
157
+ }
158
+
159
+ explicit FP32Vec8(const float *ptr) {
160
+ reg.val[0] = vec_xl(0, ptr);
161
+ reg.val[1] = vec_xl(16, ptr);
162
+ }
163
+
164
+ explicit FP32Vec8(f32x4x2_t data) : reg(data) {}
165
+
166
+ explicit FP32Vec8(const FP32Vec8 &data) {
167
+ reg.val[0] = data.reg.val[0];
168
+ reg.val[1] = data.reg.val[1];
169
+ }
170
+
171
+ explicit FP32Vec8(const BF16Vec8 &v) {
172
+ reg.val[0] = (__vector float)vec_mergeh(zero, v.reg);
173
+ reg.val[1] = (__vector float)vec_mergel(zero, v.reg);
174
+ }
175
+
176
+ float reduce_sum() const {
177
+ AliasReg ar;
178
+ ar.reg = reg;
179
+ float result = 0;
180
+ unroll_loop<int, VEC_ELEM_NUM>([&result, &ar](int i) { result += ar.values[i]; });
181
+
182
+ return result;
183
+ }
184
+
185
+ FP32Vec8 exp() const {
186
+ // TODO: Vectorize this
187
+ AliasReg ar;
188
+ ar.reg = reg;
189
+ f32x4x4_t ret;
190
+ ret.val[0][0] = std::exp(ar.values[0]);
191
+ ret.val[0][1] = std::exp(ar.values[1]);
192
+ ret.val[0][2] = std::exp(ar.values[2]);
193
+ ret.val[0][3] = std::exp(ar.values[3]);
194
+ ret.val[1][0] = std::exp(ar.values[4]);
195
+ ret.val[1][1] = std::exp(ar.values[5]);
196
+ ret.val[1][2] = std::exp(ar.values[6]);
197
+ ret.val[1][3] = std::exp(ar.values[7]);
198
+ return FP32Vec8(f32x4x2_t({ret.val[0], ret.val[1]}));
199
+ }
200
+
201
+ FP32Vec8 tanh() const {
202
+ // TODO: Vectorize this
203
+ AliasReg ar;
204
+ ar.reg = reg;
205
+ f32x4x4_t ret;
206
+ ret.val[0][0] = std::tanh(ar.values[0]);
207
+ ret.val[0][1] = std::tanh(ar.values[1]);
208
+ ret.val[0][2] = std::tanh(ar.values[2]);
209
+ ret.val[0][3] = std::tanh(ar.values[3]);
210
+ ret.val[1][0] = std::tanh(ar.values[4]);
211
+ ret.val[1][1] = std::tanh(ar.values[5]);
212
+ ret.val[1][2] = std::tanh(ar.values[6]);
213
+ ret.val[1][3] = std::tanh(ar.values[7]);
214
+ return FP32Vec8(f32x4x2_t({ret.val[0], ret.val[1]}));
215
+ }
216
+
217
+ FP32Vec8 er() const {
218
+ // TODO: Vectorize this
219
+ AliasReg ar;
220
+ ar.reg = reg;
221
+ f32x4x4_t ret;
222
+ ret.val[0][0] = std::erf(ar.values[0]);
223
+ ret.val[0][1] = std::erf(ar.values[1]);
224
+ ret.val[0][2] = std::erf(ar.values[2]);
225
+ ret.val[0][3] = std::erf(ar.values[3]);
226
+ ret.val[1][0] = std::erf(ar.values[4]);
227
+ ret.val[1][1] = std::erf(ar.values[5]);
228
+ ret.val[1][2] = std::erf(ar.values[6]);
229
+ ret.val[1][3] = std::erf(ar.values[7]);
230
+ return FP32Vec8(f32x4x2_t({ret.val[0], ret.val[1]}));
231
+ }
232
+
233
+ FP32Vec8 operator*(const FP32Vec8 &b) const {
234
+ return FP32Vec8({vec_mul(reg.val[0], b.reg.val[0]), vec_mul(reg.val[1], b.reg.val[1])});
235
+ }
236
+
237
+ FP32Vec8 operator+(const FP32Vec8 &b) const {
238
+ return FP32Vec8({vec_add(reg.val[0], b.reg.val[0]), vec_add(reg.val[1], b.reg.val[1])});
239
+ }
240
+
241
+ FP32Vec8 operator-(const FP32Vec8 &b) const {
242
+ return FP32Vec8({vec_sub(reg.val[0], b.reg.val[0]), vec_sub(reg.val[1], b.reg.val[1])});
243
+ }
244
+
245
+ FP32Vec8 operator/(const FP32Vec8 &b) const {
246
+ return FP32Vec8({vec_div(reg.val[0], b.reg.val[0]), vec_div(reg.val[1], b.reg.val[1])});
247
+ }
248
+
249
+ void save(float *ptr) const {
250
+ vec_xst(reg.val[0], 0, ptr);
251
+ vec_xst(reg.val[1], 16, ptr);
252
+ }
253
+};
254
+
255
+struct FP32Vec16 : public Vec<FP32Vec16> {
256
+ constexpr static int VEC_ELEM_NUM = 16;
257
+ union AliasReg {
258
+ f32x4x4_t reg;
259
+ float values[VEC_ELEM_NUM];
260
+ };
261
+
262
+ f32x4x4_t reg;
263
+
264
+ explicit FP32Vec16(float v) {
265
+ reg.val[0] = vec_splats(v);
266
+ reg.val[1] = vec_splats(v);
267
+ reg.val[2] = vec_splats(v);
268
+ reg.val[3] = vec_splats(v);
269
+ }
270
+
271
+ explicit FP32Vec16() {
272
+ reg.val[0] = vec_splats(0.0f);
273
+ reg.val[1] = vec_splats(0.0f);
274
+ reg.val[2] = vec_splats(0.0f);
275
+ reg.val[3] = vec_splats(0.0f);
276
+ }
277
+
278
+ explicit FP32Vec16(const float *ptr) {
279
+ reg.val[0] = vec_xl(0, ptr);
280
+ reg.val[1] = vec_xl(16, ptr);
281
+ reg.val[2] = vec_xl(32, ptr);
282
+ reg.val[3] = vec_xl(48, ptr);
283
+ }
284
+
285
+ explicit FP32Vec16(f32x4x4_t data) : reg(data) {}
286
+
287
+ explicit FP32Vec16(const FP32Vec16 &data) {
288
+ reg.val[0] = data.reg.val[0];
289
+ reg.val[1] = data.reg.val[1];
290
+ reg.val[2] = data.reg.val[2];
291
+ reg.val[3] = data.reg.val[3];
292
+ }
293
+
294
+ explicit FP32Vec16(const FP32Vec4 &data) {
295
+ reg.val[0] = data.reg;
296
+ reg.val[1] = data.reg;
297
+ reg.val[2] = data.reg;
298
+ reg.val[3] = data.reg;
299
+ }
300
+
301
+ explicit FP32Vec16(const FP32Vec8 &data) {
302
+ reg.val[0] = data.reg.val[0];
303
+ reg.val[1] = data.reg.val[1];
304
+ reg.val[2] = data.reg.val[0];
305
+ reg.val[3] = data.reg.val[1];
306
+ }
307
+
308
+ explicit FP32Vec16(const BF16Vec16 &v) {
309
+ reg.val[0] = (__vector float)vec_mergeh(zero, v.reg.val[0]);
310
+ reg.val[1] = (__vector float)vec_mergel(zero, v.reg.val[0]);
311
+ reg.val[2] = (__vector float)vec_mergeh(zero, v.reg.val[1]);
312
+ reg.val[3] = (__vector float)vec_mergel(zero, v.reg.val[1]);
313
+ }
314
+
315
+ explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
316
+
317
+ FP32Vec16 operator*(const FP32Vec16 &b) const {
318
+ return FP32Vec16(f32x4x4_t({
319
+ vec_mul(reg.val[0], b.reg.val[0]),
320
+ vec_mul(reg.val[1], b.reg.val[1]),
321
+ vec_mul(reg.val[2], b.reg.val[2]),
322
+ vec_mul(reg.val[3], b.reg.val[3])}));
323
+ }
324
+
325
+ FP32Vec16 operator+(const FP32Vec16 &b) const {
326
+ return FP32Vec16(f32x4x4_t({
327
+ vec_add(reg.val[0], b.reg.val[0]),
328
+ vec_add(reg.val[1], b.reg.val[1]),
329
+ vec_add(reg.val[2], b.reg.val[2]),
330
+ vec_add(reg.val[3], b.reg.val[3])}));
331
+ }
332
+
333
+ FP32Vec16 operator-(const FP32Vec16 &b) const {
334
+ return FP32Vec16(f32x4x4_t({
335
+ vec_sub(reg.val[0], b.reg.val[0]),
336
+ vec_sub(reg.val[1], b.reg.val[1]),
337
+ vec_sub(reg.val[2], b.reg.val[2]),
338
+ vec_sub(reg.val[3], b.reg.val[3])}));
339
+ }
340
+
341
+ FP32Vec16 operator/(const FP32Vec16 &b) const {
342
+ return FP32Vec16(f32x4x4_t({
343
+ vec_div(reg.val[0], b.reg.val[0]),
344
+ vec_div(reg.val[1], b.reg.val[1]),
345
+ vec_div(reg.val[2], b.reg.val[2]),
346
+ vec_div(reg.val[3], b.reg.val[3])}));
347
+ }
348
+
349
+ float reduce_sum() const {
350
+ AliasReg ar;
351
+ ar.reg = reg;
352
+ float result = 0;
353
+ unroll_loop<int, VEC_ELEM_NUM>([&result, &ar](int i) { result += ar.values[i]; });
354
+
355
+ return result;
356
+ }
357
+
358
+ template <int group_size> float reduce_sub_sum(int idx) {
359
+ static_assert(VEC_ELEM_NUM % group_size == 0);
360
+
361
+ AliasReg ar;
362
+ ar.reg = reg;
363
+ float result = 0;
364
+ const int start = idx * group_size;
365
+ unroll_loop<int, group_size>(
366
+ [&result, &start, ar](int i) { result += ar.values[start + i]; });
367
+
368
+ return result;
369
+ }
370
+
371
+ void save(float *ptr) const {
372
+ vec_xst(reg.val[0], 0, ptr);
373
+ vec_xst(reg.val[1], 16, ptr);
374
+ vec_xst(reg.val[2], 32, ptr);
375
+ vec_xst(reg.val[3], 48, ptr);
376
+ }
377
+};
378
+
379
+template <typename T> struct VecType { using vec_type = void; };
380
+
381
+template <typename T> using vec_t = typename VecType<T>::vec_type;
382
+
383
+template <> struct VecType<float> { using vec_type = FP32Vec8; };
384
+
385
+template <> struct VecType<c10::BFloat16> { using vec_type = BF16Vec8; };
386
+
387
+template <typename T> void storeFP32(float v, T *ptr) { *ptr = v; }
388
+
389
+inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
390
+ acc = acc + a * b;
391
+}
392
+
393
+template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
394
+ c10::BFloat16 __attribute__((__may_alias__)) *v_ptr =
395
+ reinterpret_cast<c10::BFloat16 *>(&v);
396
+ *ptr = *(v_ptr + 1);
397
+}
398
+
399
+#ifndef __VEC_CLASS_FP_NAN
400
+#define __VEC_CLASS_FP_NAN (1 << 6)
401
+#endif
402
+
403
+const static __vector unsigned char omask = { 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29 };
404
+#ifndef _ARCH_PWR10
405
+const static __vector unsigned int bias = { 0x00007fff, 0x00007fff, 0x00007fff, 0x00007fff };
406
+const static __vector unsigned int nan = { 0x7fc00000, 0x7fc00000, 0x7fc00000, 0x7fc00000 };
407
+const static __vector unsigned int sh16 = { 16, 16, 16, 16 };
408
+const static __vector unsigned int one = { 1, 1, 1, 1 };
409
+#endif
410
+
411
+inline BF16Vec8::BF16Vec8(const FP32Vec8 &v) {
412
+#ifdef _ARCH_PWR10
413
+ __vector signed short ret[2];
414
+ ret[0] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[0]);
415
+ ret[1] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[1]);
416
+ reg = vec_perm(ret[0], ret[1], omask);
417
+#elif defined(_ARCH_PWR9)
418
+ __vector unsigned int inp0 = (__vector unsigned int)(v.reg.val[0]);
419
+ __vector unsigned int inp1 = (__vector unsigned int)(v.reg.val[1]);
420
+ __vector unsigned int lsb0 = vec_sr(inp0, sh16);
421
+ __vector unsigned int lsb1 = vec_sr(inp1, sh16);
422
+ lsb0 = vec_and(lsb0, one);
423
+ lsb1 = vec_and(lsb1, one);
424
+ __vector unsigned int rnd0 = vec_add(lsb0, bias);
425
+ __vector unsigned int rnd1 = vec_add(lsb1, bias);
426
+ inp0 = vec_add(inp0, rnd0);
427
+ inp1 = vec_add(inp1, rnd1);
428
+ __vector __bool int sel0 = vec_test_data_class(v.reg.val[0], __VEC_CLASS_FP_NAN);
429
+ __vector __bool int sel1 = vec_test_data_class(v.reg.val[1], __VEC_CLASS_FP_NAN);
430
+ inp0 = vec_sel(inp0, nan, sel0);
431
+ inp1 = vec_sel(inp1, nan, sel1);
432
+ inp0 = vec_sr(inp0, sh16);
433
+ inp1 = vec_sr(inp1, sh16);
434
+ reg = (__vector signed short)vec_perm(inp0, inp1, omask);
435
+#endif
436
+}
437
+
438
+inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
439
+#ifdef _ARCH_PWR10
440
+ __vector signed short ret[4];
441
+ ret[0] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[0]);
442
+ ret[1] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[1]);
443
+ ret[2] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[2]);
444
+ ret[3] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[3]);
445
+ reg.val[0] = vec_perm(ret[0], ret[1], omask);
446
+ reg.val[1] = vec_perm(ret[2], ret[3], omask);
447
+#elif defined(_ARCH_PWR9)
448
+ __vector unsigned int inp0 = (__vector unsigned int)(v.reg.val[0]);
449
+ __vector unsigned int inp1 = (__vector unsigned int)(v.reg.val[1]);
450
+ __vector unsigned int inp2 = (__vector unsigned int)(v.reg.val[2]);
451
+ __vector unsigned int inp3 = (__vector unsigned int)(v.reg.val[3]);
452
+ __vector unsigned int lsb0 = vec_sr(inp0, sh16);
453
+ __vector unsigned int lsb1 = vec_sr(inp1, sh16);
454
+ __vector unsigned int lsb2 = vec_sr(inp2, sh16);
455
+ __vector unsigned int lsb3 = vec_sr(inp3, sh16);
456
+ lsb0 = vec_and(lsb0, one);
457
+ lsb1 = vec_and(lsb1, one);
458
+ lsb2 = vec_and(lsb2, one);
459
+ lsb3 = vec_and(lsb3, one);
460
+ __vector unsigned int rnd0 = vec_add(lsb0, bias);
461
+ __vector unsigned int rnd1 = vec_add(lsb1, bias);
462
+ __vector unsigned int rnd2 = vec_add(lsb2, bias);
463
+ __vector unsigned int rnd3 = vec_add(lsb3, bias);
464
+ inp0 = vec_add(inp0, rnd0);
465
+ inp1 = vec_add(inp1, rnd1);
466
+ inp2 = vec_add(inp2, rnd2);
467
+ inp3 = vec_add(inp3, rnd3);
468
+ __vector __bool int sel0 = vec_test_data_class(v.reg.val[0], __VEC_CLASS_FP_NAN);
469
+ __vector __bool int sel1 = vec_test_data_class(v.reg.val[1], __VEC_CLASS_FP_NAN);
470
+ __vector __bool int sel2 = vec_test_data_class(v.reg.val[2], __VEC_CLASS_FP_NAN);
471
+ __vector __bool int sel3 = vec_test_data_class(v.reg.val[3], __VEC_CLASS_FP_NAN);
472
+ inp0 = vec_sel(inp0, nan, sel0);
473
+ inp1 = vec_sel(inp1, nan, sel1);
474
+ inp2 = vec_sel(inp2, nan, sel2);
475
+ inp3 = vec_sel(inp3, nan, sel3);
476
+ inp0 = vec_sr(inp0, sh16);
477
+ inp1 = vec_sr(inp1, sh16);
478
+ inp2 = vec_sr(inp2, sh16);
479
+ inp3 = vec_sr(inp3, sh16);
480
+ reg.val[0] = (__vector signed short)vec_perm(inp0, inp1, omask);
481
+ reg.val[1] = (__vector signed short)vec_perm(inp2, inp3, omask);
482
+#endif
483
+}
484
+
485
+inline void prefetch(const void *addr) {
486
+ __asm__ __volatile__("dcbt 0, %0" : : "r"(addr) : "memory");
487
+}
488
+
489
+}; // namespace vec_op
490
+
491
+#endif
csrc/cpu/cpu_types_x86.hpp
new
+515
@@ -0,0 +1,515 @@
1
+
2
+#ifndef CPU_TYPES_X86_HPP
3
+#define CPU_TYPES_X86_HPP
4
+
5
+#include <immintrin.h>
6
+#include <torch/all.h>
7
+
8
+#ifndef __AVX2__
9
+static_assert(false, "AVX2 must be supported for the current implementation.");
10
+#endif
11
+
12
+namespace vec_op {
13
+
14
+// FIXME: FP16 is not fully supported in Torch-CPU
15
+#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
16
+ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
17
+ AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
18
+
19
+#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
20
+ AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
21
+
22
+#ifndef CPU_OP_GUARD
23
+#define CPU_KERNEL_GUARD_IN(NAME)
24
+#define CPU_KERNEL_GUARD_OUT(NAME)
25
+#else
26
+#define CPU_KERNEL_GUARD_IN(NAME) \
27
+ std::cout << #NAME << " invoked." << std::endl;
28
+#define CPU_KERNEL_GUARD_OUT(NAME) std::cout << #NAME << " exit." << std::endl;
29
+#endif
30
+
31
+#define FORCE_INLINE __attribute__((always_inline)) inline
32
+
33
+namespace {
34
+template <typename T, T... indexes, typename F>
35
+constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F &&f) {
36
+ (f(std::integral_constant<T, indexes>{}), ...);
37
+}
38
+}; // namespace
39
+
40
+template <typename T, T count, typename F,
41
+ typename = std::enable_if_t<std::is_invocable_v<F, T>>>
42
+constexpr void unroll_loop(F &&f) {
43
+ unroll_loop_item(std::make_integer_sequence<T, count>{}, std::forward<F>(f));
44
+}
45
+
46
+template <typename T> struct Vec {
47
+ constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; }
48
+};
49
+
50
+struct FP32Vec8;
51
+struct FP32Vec16;
52
+
53
+#ifdef __AVX512FP16__
54
+struct FP16Vec8 : public Vec<FP16Vec8> {
55
+ constexpr static int VEC_ELEM_NUM = 8;
56
+
57
+ __m128h reg;
58
+
59
+ explicit FP16Vec8(_Float16 v) : reg(_mm_set1_ph(v)) {}
60
+
61
+ explicit FP16Vec8(const void *ptr) : reg(_mm_loadu_ph(ptr)) {}
62
+
63
+ explicit FP16Vec8(__m128h data) : reg(data) {}
64
+
65
+ FP16Vec8 operator*(const FP16Vec8 &b) const {
66
+ return FP16Vec8(_mm_mul_ph(reg, b.reg));
67
+ }
68
+
69
+ FP16Vec8 operator+(const FP16Vec8 &b) const {
70
+ return FP16Vec8(_mm_add_ph(reg, b.reg));
71
+ }
72
+
73
+ FP16Vec8 operator-(const FP16Vec8 &b) const {
74
+ return FP16Vec8(_mm_sub_ph(reg, b.reg));
75
+ }
76
+
77
+ FP16Vec8 operator/(const FP16Vec8 &b) const {
78
+ return FP16Vec8(_mm_div_ph(reg, b.reg));
79
+ }
80
+
81
+ void save(void *ptr) const { _mm_storeu_ph(ptr, reg); }
82
+};
83
+#endif
84
+
85
+struct BF16Vec8 : public Vec<BF16Vec8> {
86
+ constexpr static int VEC_ELEM_NUM = 8;
87
+
88
+ __m128i reg;
89
+
90
+ explicit BF16Vec8(const void *ptr)
91
+ : reg((__m128i)_mm_loadu_si128((__m128i *)ptr)) {}
92
+
93
+ explicit BF16Vec8(const FP32Vec8 &);
94
+
95
+ void save(void *ptr) const { *reinterpret_cast<__m128i *>(ptr) = reg; }
96
+};
97
+
98
+struct BF16Vec16 : public Vec<BF16Vec16> {
99
+ constexpr static int VEC_ELEM_NUM = 16;
100
+
101
+ __m256i reg;
102
+
103
+ explicit BF16Vec16(const void *ptr)
104
+ : reg((__m256i)_mm256_loadu_si256((__m256i *)ptr)) {}
105
+
106
+ explicit BF16Vec16(const FP32Vec16 &);
107
+
108
+ void save(void *ptr) const { *reinterpret_cast<__m256i *>(ptr) = reg; }
109
+};
110
+
111
+#ifdef __AVX512F__
112
+struct BF16Vec32 : public Vec<BF16Vec32> {
113
+ constexpr static int VEC_ELEM_NUM = 32;
114
+
115
+ __m512i reg;
116
+
117
+ explicit BF16Vec32(const void *ptr) : reg((__m512i)_mm512_loadu_si512(ptr)) {}
118
+
119
+ explicit BF16Vec32(__m512i data) : reg(data) {}
120
+
121
+ explicit BF16Vec32(BF16Vec8 &vec8_data)
122
+ : reg((__m512i)_mm512_inserti32x4(
123
+ _mm512_inserti32x4(_mm512_inserti32x4(_mm512_castsi128_si512(
124
+ (__m128i)vec8_data.reg),
125
+ (__m128i)vec8_data.reg, 1),
126
+ (__m128i)vec8_data.reg, 2),
127
+ (__m128i)vec8_data.reg, 3)) {}
128
+
129
+ void save(void *ptr) const { *reinterpret_cast<__m512i *>(ptr) = reg; }
130
+};
131
+#else
132
+struct BF16Vec32 : public Vec<BF16Vec32> {
133
+ constexpr static int VEC_ELEM_NUM = 32;
134
+
135
+ __m256i reg_low;
136
+ __m256i reg_high;
137
+
138
+ explicit BF16Vec32(const void *ptr)
139
+ : reg_low(_mm256_loadu_si256((__m256i const *)ptr)),
140
+ reg_high(_mm256_loadu_si256((__m256i const *)ptr + 1)) {}
141
+
142
+ explicit BF16Vec32(__m256i low, __m256i high) : reg_low(low),
143
+ reg_high(high) {}
144
+
145
+ explicit BF16Vec32(BF16Vec8 &vec8_data)
146
+ : reg_low((__m256i)_mm256_inserti32x4(
147
+ _mm256_castsi128_si256((__m128i)vec8_data.reg),
148
+ (__m128i)vec8_data.reg, 1)),
149
+ reg_high((__m256i)_mm256_inserti32x4(
150
+ _mm256_castsi128_si256((__m128i)vec8_data.reg),
151
+ (__m128i)vec8_data.reg, 1)) {}
152
+
153
+ void save(void *ptr) const {
154
+ *reinterpret_cast<__m256i *>(ptr) = reg_low;
155
+ *reinterpret_cast<__m256i *>((__m256i *)ptr + 1) = reg_high;
156
+ }
157
+};
158
+#endif
159
+
160
+struct FP32Vec4 : public Vec<FP32Vec4> {
161
+ constexpr static int VEC_ELEM_NUM = 4;
162
+ union AliasReg {
163
+ __m128 reg;
164
+ float values[VEC_ELEM_NUM];
165
+ };
166
+
167
+ __m128 reg;
168
+
169
+ explicit FP32Vec4(float v) : reg(_mm_set1_ps(v)) {}
170
+
171
+ explicit FP32Vec4() : reg(_mm_set1_ps(0.0)) {}
172
+
173
+ explicit FP32Vec4(const float *ptr) : reg(_mm_loadu_ps(ptr)) {}
174
+
175
+ explicit FP32Vec4(__m128 data) : reg(data) {}
176
+
177
+ explicit FP32Vec4(const FP32Vec4 &data) : reg(data.reg) {}
178
+};
179
+
180
+struct FP32Vec8 : public Vec<FP32Vec8> {
181
+ constexpr static int VEC_ELEM_NUM = 8;
182
+ union AliasReg {
183
+ __m256 reg;
184
+ float values[VEC_ELEM_NUM];
185
+ };
186
+
187
+ __m256 reg;
188
+
189
+ explicit FP32Vec8(float v) : reg(_mm256_set1_ps(v)) {}
190
+
191
+ explicit FP32Vec8() : reg(_mm256_set1_ps(0.0)) {}
192
+
193
+ explicit FP32Vec8(const float *ptr) : reg(_mm256_loadu_ps(ptr)) {}
194
+
195
+ explicit FP32Vec8(__m256 data) : reg(data) {}
196
+
197
+ explicit FP32Vec8(const FP32Vec8 &data) : reg(data.reg) {}
198
+
199
+#ifdef __AVX512FP16__
200
+ explicit FP32Vec8(__m128h v) : reg(_mm256_cvtph_ps(_mm_castph_si128(v))) {}
201
+#endif
202
+
203
+ explicit FP32Vec8(const BF16Vec8 &v)
204
+ : reg(_mm256_castsi256_ps(
205
+ _mm256_bslli_epi128(_mm256_cvtepu16_epi32(v.reg), 2))) {}
206
+
207
+ float reduce_sum() const {
208
+ AliasReg ar;
209
+ ar.reg = reg;
210
+ float result = 0;
211
+ unroll_loop<int, VEC_ELEM_NUM>([&result, &ar](int i) { result += ar.values[i]; });
212
+
213
+ return result;
214
+ }
215
+
216
+ FP32Vec8 exp() const {
217
+ AliasReg ar;
218
+ ar.reg = reg;
219
+ return FP32Vec8(_mm256_set_ps(expf(ar.values[7]), expf(ar.values[6]),
220
+ expf(ar.values[5]), expf(ar.values[4]),
221
+ expf(ar.values[3]), expf(ar.values[2]),
222
+ expf(ar.values[1]), expf(ar.values[0])));
223
+ }
224
+
225
+ FP32Vec8 tanh() const {
226
+ AliasReg ar;
227
+ ar.reg = reg;
228
+ return FP32Vec8(_mm256_set_ps(tanhf(ar.values[7]), tanhf(ar.values[6]),
229
+ tanhf(ar.values[5]), tanhf(ar.values[4]),
230
+ tanhf(ar.values[3]), tanhf(ar.values[2]),
231
+ tanhf(ar.values[1]), tanhf(ar.values[0])));
232
+ }
233
+
234
+ FP32Vec8 er() const {
235
+ AliasReg ar;
236
+ ar.reg = reg;
237
+ return FP32Vec8(_mm256_set_ps(erf(ar.values[7]), erf(ar.values[6]),
238
+ erf(ar.values[5]), erf(ar.values[4]),
239
+ erf(ar.values[3]), erf(ar.values[2]),
240
+ erf(ar.values[1]), erf(ar.values[0])));
241
+ }
242
+
243
+ FP32Vec8 operator*(const FP32Vec8 &b) const {
244
+ return FP32Vec8(_mm256_mul_ps(reg, b.reg));
245
+ }
246
+
247
+ FP32Vec8 operator+(const FP32Vec8 &b) const {
248
+ return FP32Vec8(_mm256_add_ps(reg, b.reg));
249
+ }
250
+
251
+ FP32Vec8 operator-(const FP32Vec8 &b) const {
252
+ return FP32Vec8(_mm256_sub_ps(reg, b.reg));
253
+ }
254
+
255
+ FP32Vec8 operator/(const FP32Vec8 &b) const {
256
+ return FP32Vec8(_mm256_div_ps(reg, b.reg));
257
+ }
258
+
259
+ void save(float *ptr) const { _mm256_storeu_ps(ptr, reg); }
260
+};
261
+
262
+#ifdef __AVX512F__
263
+struct FP32Vec16 : public Vec<FP32Vec16> {
264
+ constexpr static int VEC_ELEM_NUM = 16;
265
+ union AliasReg {
266
+ __m512 reg;
267
+ float values[VEC_ELEM_NUM];
268
+ };
269
+
270
+ __m512 reg;
271
+
272
+ explicit FP32Vec16(float v) : reg(_mm512_set1_ps(v)) {}
273
+
274
+ explicit FP32Vec16() : reg(_mm512_set1_ps(0.0)) {}
275
+
276
+ explicit FP32Vec16(const float *ptr) : reg(_mm512_loadu_ps(ptr)) {}
277
+
278
+ explicit FP32Vec16(__m512 data) : reg(data) {}
279
+
280
+ explicit FP32Vec16(const FP32Vec16 &data) : reg(data.reg) {}
281
+
282
+ explicit FP32Vec16(const FP32Vec4 &data)
283
+ : reg((__m512)_mm512_inserti32x4(
284
+ _mm512_inserti32x4(
285
+ _mm512_inserti32x4(_mm512_castsi128_si512((__m128i)data.reg),
286
+ (__m128i)data.reg, 1),
287
+ (__m128i)data.reg, 2),
288
+ (__m128i)data.reg, 3)) {}
289
+
290
+ explicit FP32Vec16(const FP32Vec8 &data)
291
+ : reg((__m512)_mm512_inserti32x8(
292
+ _mm512_castsi256_si512((__m256i)data.reg), (__m256i)data.reg, 1)) {}
293
+
294
+ explicit FP32Vec16(const BF16Vec16 &v)
295
+ : reg(_mm512_castsi512_ps(
296
+ _mm512_bslli_epi128(_mm512_cvtepu16_epi32(v.reg), 2))) {}
297
+
298
+ explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
299
+
300
+ FP32Vec16 operator*(const FP32Vec16 &b) const {
301
+ return FP32Vec16(_mm512_mul_ps(reg, b.reg));
302
+ }
303
+
304
+ FP32Vec16 operator+(const FP32Vec16 &b) const {
305
+ return FP32Vec16(_mm512_add_ps(reg, b.reg));
306
+ }
307
+
308
+ FP32Vec16 operator-(const FP32Vec16 &b) const {
309
+ return FP32Vec16(_mm512_sub_ps(reg, b.reg));
310
+ }
311
+
312
+ FP32Vec16 operator/(const FP32Vec16 &b) const {
313
+ return FP32Vec16(_mm512_div_ps(reg, b.reg));
314
+ }
315
+
316
+ float reduce_sum() const { return _mm512_reduce_add_ps(reg); }
317
+
318
+ template <int group_size> float reduce_sub_sum(int idx) {
319
+ static_assert(VEC_ELEM_NUM % group_size == 0);
320
+ constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
321
+ __mmask16 mask = _cvtu32_mask16(base_mask << (idx * group_size));
322
+ return _mm512_mask_reduce_add_ps(mask, reg);
323
+ }
324
+
325
+ void save(float *ptr) const { _mm512_storeu_ps(ptr, reg); }
326
+};
327
+#else
328
+struct FP32Vec16 : public Vec<FP32Vec16> {
329
+ constexpr static int VEC_ELEM_NUM = 16;
330
+
331
+ union AliasReg {
332
+ __m256 reg;
333
+ float values[8];
334
+ };
335
+
336
+ __m256 reg_low;
337
+ __m256 reg_high;
338
+
339
+ explicit FP32Vec16(float v) : reg_low(_mm256_set1_ps(v)),
340
+ reg_high(_mm256_set1_ps(v)) {}
341
+
342
+ explicit FP32Vec16() : reg_low(_mm256_set1_ps(0.0)),
343
+ reg_high(_mm256_set1_ps(0.0)) {}
344
+
345
+ explicit FP32Vec16(const float *ptr) : reg_low(_mm256_loadu_ps(ptr)),
346
+ reg_high(_mm256_loadu_ps(ptr + 8)) {}
347
+
348
+ explicit FP32Vec16(__m256 low, __m256 high) : reg_low(low), reg_high(high) {}
349
+
350
+ explicit FP32Vec16(const FP32Vec16 &data) : reg_low(data.reg_low),
351
+ reg_high(data.reg_high) {}
352
+
353
+ explicit FP32Vec16(const FP32Vec4 &data)
354
+ : reg_low((__m256)_mm256_inserti128_si256(
355
+ _mm256_castsi128_si256((__m128i)data.reg),
356
+ (__m128i)data.reg, 1)),
357
+ reg_high((__m256)_mm256_inserti128_si256(
358
+ _mm256_castsi128_si256((__m128i)data.reg),
359
+ (__m128i)data.reg, 1)) {}
360
+
361
+ explicit FP32Vec16(const FP32Vec8 &data)
362
+ : reg_low(data.reg), reg_high(data.reg) {}
363
+
364
+ explicit FP32Vec16(const BF16Vec16 &v) {
365
+ __m128i low = _mm256_extractf128_si256(v.reg, 0);
366
+ __m128i high = _mm256_extractf128_si256(v.reg, 1);
367
+
368
+ __m256i v_low_epi32 = _mm256_cvtepu16_epi32(low);
369
+ __m256i v_high_epi32 = _mm256_cvtepu16_epi32(high);
370
+
371
+ __m256i v_low_shifted = _mm256_bslli_epi128(v_low_epi32, 2);
372
+ __m256i v_high_shifted = _mm256_bslli_epi128(v_high_epi32, 2);
373
+
374
+ reg_low = _mm256_castsi256_ps(v_low_shifted);
375
+ reg_high = _mm256_castsi256_ps(v_high_shifted);
376
+ }
377
+
378
+ explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
379
+
380
+ FP32Vec16 operator*(const FP32Vec16 &b) const {
381
+ return FP32Vec16(_mm256_mul_ps(reg_low, b.reg_low),
382
+ _mm256_mul_ps(reg_high, b.reg_high));
383
+ }
384
+
385
+ FP32Vec16 operator+(const FP32Vec16 &b) const {
386
+ return FP32Vec16(_mm256_add_ps(reg_low, b.reg_low),
387
+ _mm256_add_ps(reg_high, b.reg_high));
388
+ }
389
+
390
+ FP32Vec16 operator-(const FP32Vec16 &b) const {
391
+ return FP32Vec16(_mm256_sub_ps(reg_low, b.reg_low),
392
+ _mm256_sub_ps(reg_high, b.reg_high));
393
+ }
394
+
395
+ FP32Vec16 operator/(const FP32Vec16 &b) const {
396
+ return FP32Vec16(_mm256_div_ps(reg_low, b.reg_low),
397
+ _mm256_div_ps(reg_high, b.reg_high));
398
+ }
399
+
400
+ float reduce_sum() const {
401
+ FP32Vec8 low = FP32Vec8(reg_low);
402
+ FP32Vec8 high = FP32Vec8(reg_high);
403
+ return low.reduce_sum() + high.reduce_sum();
404
+ }
405
+
406
+ template <int group_size> float reduce_sub_sum(int idx) {
407
+ float sum = 0.0;
408
+ static_assert(VEC_ELEM_NUM % group_size == 0);
409
+ constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
410
+ uint32_t mask = base_mask << (idx * group_size);
411
+
412
+ AliasReg ar;
413
+
414
+ auto func = [&sum, &mask, &ar](int i) {
415
+ int flag = mask & 0x1;
416
+ mask = mask >> 1;
417
+ if (flag != 0) sum += ar.values[i];
418
+ };
419
+
420
+ ar.reg = reg_low;
421
+ unroll_loop<int, 8>(func);
422
+
423
+ ar.reg = reg_high;
424
+ unroll_loop<int, 8>(func);
425
+
426
+ return sum;
427
+ }
428
+
429
+ void save(float *ptr) const {
430
+ _mm256_storeu_ps(ptr, reg_low);
431
+ _mm256_storeu_ps(ptr + 8, reg_high);
432
+ }
433
+};
434
+#endif
435
+
436
+template <typename T> struct VecType { using vec_type = void; };
437
+
438
+template <typename T> using vec_t = typename VecType<T>::vec_type;
439
+
440
+template <> struct VecType<float> { using vec_type = FP32Vec8; };
441
+
442
+#ifdef __AVX512FP16__
443
+template <> struct VecType<c10::Half> { using vec_type = FP16Vec16; };
444
+#endif
445
+
446
+template <> struct VecType<c10::BFloat16> { using vec_type = BF16Vec8; };
447
+
448
+template <typename T> void storeFP32(float v, T *ptr) { *ptr = v; }
449
+
450
+#ifdef __AVX512FP16__
451
+template <> inline void storeFP32<c10::Half>(float v, c10::Half *ptr) {
452
+ *reinterpret_cast<_Float16 *>(ptr) = v;
453
+}
454
+#endif
455
+
456
+inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
457
+ acc = acc + a * b;
458
+}
459
+
460
+#ifdef __AVX512BF16__
461
+template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
462
+ *reinterpret_cast<__bfloat16 *>(ptr) = _mm_cvtness_sbh(v);
463
+}
464
+
465
+inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
466
+ : reg((__m128i)_mm256_cvtneps_pbh(v.reg)) {}
467
+
468
+inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
469
+ : reg((__m256i)_mm512_cvtneps_pbh(v.reg)) {}
470
+
471
+inline void fma(FP32Vec16 &acc, BF16Vec32 &a, BF16Vec32 &b) {
472
+ acc.reg = _mm512_dpbf16_ps(acc.reg, (__m512bh)a.reg, (__m512bh)b.reg);
473
+}
474
+#else
475
+template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
476
+ c10::BFloat16 __attribute__((__may_alias__)) *v_ptr =
477
+ reinterpret_cast<c10::BFloat16 *>(&v);
478
+ *ptr = *(v_ptr + 1);
479
+}
480
+
481
+#ifdef __AVX512F__
482
+inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
483
+ : reg(_mm256_cvtepi32_epi16(
484
+ _mm256_bsrli_epi128(_mm256_castps_si256(v.reg), 2))) {}
485
+
486
+inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
487
+ : reg(_mm512_cvtepi32_epi16(
488
+ _mm512_bsrli_epi128(_mm512_castps_si512(v.reg), 2))) {}
489
+#else
490
+namespace{
491
+__m128i FP32Vec8_to_BF16Vec8_avx2(__m256 a) {
492
+ __m256i ai = _mm256_castps_si256(a);
493
+ ai = _mm256_srli_epi32(ai, 16);
494
+ ai = _mm256_packus_epi32(ai, ai);
495
+ ai = _mm256_permute4x64_epi64(ai, 0b00111001);
496
+ return _mm256_extracti128_si256(ai, 0);
497
+}
498
+}
499
+
500
+inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
501
+ : reg(FP32Vec8_to_BF16Vec8_avx2(v.reg)) {}
502
+
503
+inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
504
+ BF16Vec8 low = BF16Vec8(FP32Vec8(v.reg_low));
505
+ BF16Vec8 high = BF16Vec8(FP32Vec8(v.reg_high));
506
+ reg = _mm256_insertf128_si256(_mm256_castsi128_si256(low.reg), high.reg, 1);
507
+}
508
+#endif // __AVX512F__
509
+#endif // __AVX512BF16__
510
+
511
+inline void prefetch(const void *addr) { _mm_prefetch(addr, _MM_HINT_T1); }
512
+
513
+}; // namespace vec_op
514
+
515
+#endif
csrc/ops.h
+1
@@ -1,5 +1,6 @@
1
#pragma once
2
3
+#include <optional>
4
#include <torch/library.h>
5
6
void paged_attention_v1(
requirements-cpu.txt
+3
-3
@@ -2,6 +2,6 @@
2
-r requirements-common.txt
3
4
# Dependencies for x86_64 CPUs
5
-torch == 2.3.1+cpu
6
-torchvision == 0.18.1+cpu # required for the image processor of phi3v, this must be updated alongside torch
7
-triton >= 2.2.0 # FIXME(woosuk): This is a hack to avoid import error.
\ No newline at end of file
5
+torch == 2.3.1+cpu; platform_machine != "ppc64le"
6
+torchvision == 0.18.1+cpu; platform_machine != "ppc64le" # required for the image processor of phi3v, this must be updated alongside torch
7
+triton >= 2.2.0 # FIXME(woosuk): This is a hack to avoid import error.