A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Hardware][Intel] Support CPU inference with AVX2 ISA (#5452)
Jie Fu (傅杰) committed
Jun 14, 2024 at 07:22 UTC
cd9c0d65d98f86fbd2235ee41b80107097a57f77
2 files changed
+169
-2
cmake/cpu_extension.cmake
+5
-1
@@ -33,6 +33,7 @@ function (find_isa CPUINFO TARGET OUT)
33
endif()
34
endfunction()
35
36
+find_isa(${CPUINFO} "avx2" AVX2_FOUND)
37
find_isa(${CPUINFO} "avx512f" AVX512_FOUND)
38
39
if (AVX512_FOUND)
@@ -53,8 +54,11 @@ if (AVX512_FOUND)
54
else()
55
message(WARNING "Disable AVX512-BF16 ISA support, no avx512_bf16 found in local CPU flags." " If cross-compilation is required, please set env VLLM_CPU_AVX512BF16=1.")
56
endif()
57
+elseif (AVX2_FOUND)
58
+ list(APPEND CXX_COMPILE_FLAGS "-mavx2")
59
+ message(WARNING "vLLM CPU backend using AVX2 ISA")
60
else()
57
- message(FATAL_ERROR "vLLM CPU backend requires AVX512 ISA support.")
61
+ message(FATAL_ERROR "vLLM CPU backend requires AVX512 or AVX2 ISA support.")
62
endif()
63
64
message(STATUS "CPU extension compile flags: ${CXX_COMPILE_FLAGS}")
csrc/cpu/cpu_types.hpp
+164
-1
@@ -5,6 +5,10 @@
5
#include <immintrin.h>
6
#include <torch/all.h>
7
8
+#ifndef __AVX2__
9
+static_assert(false, "AVX2 must be supported for the current implementation.");
10
+#endif
11
+
12
namespace vec_op {
13
14
// FIXME: FP16 is not fully supported in Torch-CPU
@@ -104,6 +108,7 @@ struct BF16Vec16 : public Vec<BF16Vec16> {
108
void save(void *ptr) const { *reinterpret_cast<__m256i *>(ptr) = reg; }
109
};
110
111
+#ifdef __AVX512F__
112
struct BF16Vec32 : public Vec<BF16Vec32> {
113
constexpr static int VEC_ELEM_NUM = 32;
114
@@ -123,6 +128,34 @@ struct BF16Vec32 : public Vec<BF16Vec32> {
128
129
void save(void *ptr) const { *reinterpret_cast<__m512i *>(ptr) = reg; }
130
};
131
+#else
132
+struct BF16Vec32 : public Vec<BF16Vec32> {
133
+ constexpr static int VEC_ELEM_NUM = 32;
134
+
135
+ __m256i reg_low;
136
+ __m256i reg_high;
137
+
138
+ explicit BF16Vec32(const void *ptr)
139
+ : reg_low(_mm256_loadu_si256((__m256i const *)ptr)),
140
+ reg_high(_mm256_loadu_si256((__m256i const *)ptr + 1)) {}
141
+
142
+ explicit BF16Vec32(__m256i low, __m256i high) : reg_low(low),
143
+ reg_high(high) {}
144
+
145
+ explicit BF16Vec32(BF16Vec8 &vec8_data)
146
+ : reg_low((__m256i)_mm256_inserti32x4(
147
+ _mm256_castsi128_si256((__m128i)vec8_data.reg),
148
+ (__m128i)vec8_data.reg, 1)),
149
+ reg_high((__m256i)_mm256_inserti32x4(
150
+ _mm256_castsi128_si256((__m128i)vec8_data.reg),
151
+ (__m128i)vec8_data.reg, 1)) {}
152
+
153
+ void save(void *ptr) const {
154
+ *reinterpret_cast<__m256i *>(ptr) = reg_low;
155
+ *reinterpret_cast<__m256i *>((__m256i *)ptr + 1) = reg_high;
156
+ }
157
+};
158
+#endif
159
160
struct FP32Vec4 : public Vec<FP32Vec4> {
161
constexpr static int VEC_ELEM_NUM = 4;
@@ -226,6 +259,7 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
259
void save(float *ptr) const { _mm256_storeu_ps(ptr, reg); }
260
};
261
262
+#ifdef __AVX512F__
263
struct FP32Vec16 : public Vec<FP32Vec16> {
264
constexpr static int VEC_ELEM_NUM = 16;
265
union AliasReg {
@@ -290,6 +324,114 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
324
325
void save(float *ptr) const { _mm512_storeu_ps(ptr, reg); }
326
};
327
+#else
328
+struct FP32Vec16 : public Vec<FP32Vec16> {
329
+ constexpr static int VEC_ELEM_NUM = 16;
330
+
331
+ union AliasReg {
332
+ __m256 reg;
333
+ float values[8];
334
+ };
335
+
336
+ __m256 reg_low;
337
+ __m256 reg_high;
338
+
339
+ explicit FP32Vec16(float v) : reg_low(_mm256_set1_ps(v)),
340
+ reg_high(_mm256_set1_ps(v)) {}
341
+
342
+ explicit FP32Vec16() : reg_low(_mm256_set1_ps(0.0)),
343
+ reg_high(_mm256_set1_ps(0.0)) {}
344
+
345
+ explicit FP32Vec16(const float *ptr) : reg_low(_mm256_loadu_ps(ptr)),
346
+ reg_high(_mm256_loadu_ps(ptr + 8)) {}
347
+
348
+ explicit FP32Vec16(__m256 low, __m256 high) : reg_low(low), reg_high(high) {}
349
+
350
+ explicit FP32Vec16(const FP32Vec16 &data) : reg_low(data.reg_low),
351
+ reg_high(data.reg_high) {}
352
+
353
+ explicit FP32Vec16(const FP32Vec4 &data)
354
+ : reg_low((__m256)_mm256_inserti128_si256(
355
+ _mm256_castsi128_si256((__m128i)data.reg),
356
+ (__m128i)data.reg, 1)),
357
+ reg_high((__m256)_mm256_inserti128_si256(
358
+ _mm256_castsi128_si256((__m128i)data.reg),
359
+ (__m128i)data.reg, 1)) {}
360
+
361
+ explicit FP32Vec16(const FP32Vec8 &data)
362
+ : reg_low(data.reg), reg_high(data.reg) {}
363
+
364
+ explicit FP32Vec16(const BF16Vec16 &v) {
365
+ __m128i low = _mm256_extractf128_si256(v.reg, 0);
366
+ __m128i high = _mm256_extractf128_si256(v.reg, 1);
367
+
368
+ __m256i v_low_epi32 = _mm256_cvtepu16_epi32(low);
369
+ __m256i v_high_epi32 = _mm256_cvtepu16_epi32(high);
370
+
371
+ __m256i v_low_shifted = _mm256_bslli_epi128(v_low_epi32, 2);
372
+ __m256i v_high_shifted = _mm256_bslli_epi128(v_high_epi32, 2);
373
+
374
+ reg_low = _mm256_castsi256_ps(v_low_shifted);
375
+ reg_high = _mm256_castsi256_ps(v_high_shifted);
376
+ }
377
+
378
+ explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
379
+
380
+ FP32Vec16 operator*(const FP32Vec16 &b) const {
381
+ return FP32Vec16(_mm256_mul_ps(reg_low, b.reg_low),
382
+ _mm256_mul_ps(reg_high, b.reg_high));
383
+ }
384
+
385
+ FP32Vec16 operator+(const FP32Vec16 &b) const {
386
+ return FP32Vec16(_mm256_add_ps(reg_low, b.reg_low),
387
+ _mm256_add_ps(reg_high, b.reg_high));
388
+ }
389
+
390
+ FP32Vec16 operator-(const FP32Vec16 &b) const {
391
+ return FP32Vec16(_mm256_sub_ps(reg_low, b.reg_low),
392
+ _mm256_sub_ps(reg_high, b.reg_high));
393
+ }
394
+
395
+ FP32Vec16 operator/(const FP32Vec16 &b) const {
396
+ return FP32Vec16(_mm256_div_ps(reg_low, b.reg_low),
397
+ _mm256_div_ps(reg_high, b.reg_high));
398
+ }
399
+
400
+ float reduce_sum() const {
401
+ FP32Vec8 low = FP32Vec8(reg_low);
402
+ FP32Vec8 high = FP32Vec8(reg_high);
403
+ return low.reduce_sum() + high.reduce_sum();
404
+ }
405
+
406
+ template <int group_size> float reduce_sub_sum(int idx) {
407
+ float sum = 0.0;
408
+ static_assert(VEC_ELEM_NUM % group_size == 0);
409
+ constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
410
+ uint32_t mask = base_mask << (idx * group_size);
411
+
412
+ AliasReg ar;
413
+
414
+ auto func = [&sum, &mask, &ar](int i) {
415
+ int flag = mask & 0x1;
416
+ mask = mask >> 1;
417
+ if (flag != 0) sum += ar.values[i];
418
+ };
419
+
420
+ ar.reg = reg_low;
421
+ unroll_loop<int, 8>(func);
422
+
423
+ ar.reg = reg_high;
424
+ unroll_loop<int, 8>(func);
425
+
426
+ return sum;
427
+ }
428
+
429
+ void save(float *ptr) const {
430
+ _mm256_storeu_ps(ptr, reg_low);
431
+ _mm256_storeu_ps(ptr + 8, reg_high);
432
+ }
433
+};
434
+#endif
435
436
template <typename T> struct VecType { using vec_type = void; };
437
@@ -336,6 +478,7 @@ template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
478
*ptr = *(v_ptr + 1);
479
}
480
481
+#ifdef __AVX512F__
482
inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
483
: reg(_mm256_cvtepi32_epi16(
484
_mm256_bsrli_epi128(_mm256_castps_si256(v.reg), 2))) {}
@@ -343,7 +486,27 @@ inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
486
inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
487
: reg(_mm512_cvtepi32_epi16(
488
_mm512_bsrli_epi128(_mm512_castps_si512(v.reg), 2))) {}
346
-#endif
489
+#else
490
+namespace{
491
+__m128i FP32Vec8_to_BF16Vec8_avx2(__m256 a) {
492
+ __m256i ai = _mm256_castps_si256(a);
493
+ ai = _mm256_srli_epi32(ai, 16);
494
+ ai = _mm256_packus_epi32(ai, ai);
495
+ ai = _mm256_permute4x64_epi64(ai, 0b00111001);
496
+ return _mm256_extracti128_si256(ai, 0);
497
+}
498
+}
499
+
500
+inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
501
+ : reg(FP32Vec8_to_BF16Vec8_avx2(v.reg)) {}
502
+
503
+inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
504
+ BF16Vec8 low = BF16Vec8(FP32Vec8(v.reg_low));
505
+ BF16Vec8 high = BF16Vec8(FP32Vec8(v.reg_high));
506
+ reg = _mm256_insertf128_si256(_mm256_castsi128_si256(low.reg), high.reg, 1);
507
+}
508
+#endif // __AVX512F__
509
+#endif // __AVX512BF16__
510
511
inline void prefetch(const void *addr) { _mm_prefetch(addr, _MM_HINT_T1); }
512