@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cd9c0d65d9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Hardware][Intel] Support CPU inference with AVX2 ISA (#5452)

Jie Fu (傅杰) committed Jun 14, 2024 at 07:22 UTC cd9c0d65d98f86fbd2235ee41b80107097a57f77
2 files changed +169 -2
cmake/cpu_extension.cmake
+5 -1
@@ -33,6 +33,7 @@ function (find_isa CPUINFO TARGET OUT)
33 endif()
34 endfunction()
35
36 +find_isa(${CPUINFO} "avx2" AVX2_FOUND)
37 find_isa(${CPUINFO} "avx512f" AVX512_FOUND)
38
39 if (AVX512_FOUND)
@@ -53,8 +54,11 @@ if (AVX512_FOUND)
54 else()
55 message(WARNING "Disable AVX512-BF16 ISA support, no avx512_bf16 found in local CPU flags." " If cross-compilation is required, please set env VLLM_CPU_AVX512BF16=1.")
56 endif()
57 +elseif (AVX2_FOUND)
58 + list(APPEND CXX_COMPILE_FLAGS "-mavx2")
59 + message(WARNING "vLLM CPU backend using AVX2 ISA")
60 else()
57 - message(FATAL_ERROR "vLLM CPU backend requires AVX512 ISA support.")
61 + message(FATAL_ERROR "vLLM CPU backend requires AVX512 or AVX2 ISA support.")
62 endif()
63
64 message(STATUS "CPU extension compile flags: ${CXX_COMPILE_FLAGS}")
csrc/cpu/cpu_types.hpp
+164 -1
@@ -5,6 +5,10 @@
5 #include <immintrin.h>
6 #include <torch/all.h>
7
8 +#ifndef __AVX2__
9 +static_assert(false, "AVX2 must be supported for the current implementation.");
10 +#endif
11 +
12 namespace vec_op {
13
14 // FIXME: FP16 is not fully supported in Torch-CPU
@@ -104,6 +108,7 @@ struct BF16Vec16 : public Vec<BF16Vec16> {
108 void save(void *ptr) const { *reinterpret_cast<__m256i *>(ptr) = reg; }
109 };
110
111 +#ifdef __AVX512F__
112 struct BF16Vec32 : public Vec<BF16Vec32> {
113 constexpr static int VEC_ELEM_NUM = 32;
114
@@ -123,6 +128,34 @@ struct BF16Vec32 : public Vec<BF16Vec32> {
128
129 void save(void *ptr) const { *reinterpret_cast<__m512i *>(ptr) = reg; }
130 };
131 +#else
132 +struct BF16Vec32 : public Vec<BF16Vec32> {
133 + constexpr static int VEC_ELEM_NUM = 32;
134 +
135 + __m256i reg_low;
136 + __m256i reg_high;
137 +
138 + explicit BF16Vec32(const void *ptr)
139 + : reg_low(_mm256_loadu_si256((__m256i const *)ptr)),
140 + reg_high(_mm256_loadu_si256((__m256i const *)ptr + 1)) {}
141 +
142 + explicit BF16Vec32(__m256i low, __m256i high) : reg_low(low),
143 + reg_high(high) {}
144 +
145 + explicit BF16Vec32(BF16Vec8 &vec8_data)
146 + : reg_low((__m256i)_mm256_inserti32x4(
147 + _mm256_castsi128_si256((__m128i)vec8_data.reg),
148 + (__m128i)vec8_data.reg, 1)),
149 + reg_high((__m256i)_mm256_inserti32x4(
150 + _mm256_castsi128_si256((__m128i)vec8_data.reg),
151 + (__m128i)vec8_data.reg, 1)) {}
152 +
153 + void save(void *ptr) const {
154 + *reinterpret_cast<__m256i *>(ptr) = reg_low;
155 + *reinterpret_cast<__m256i *>((__m256i *)ptr + 1) = reg_high;
156 + }
157 +};
158 +#endif
159
160 struct FP32Vec4 : public Vec<FP32Vec4> {
161 constexpr static int VEC_ELEM_NUM = 4;
@@ -226,6 +259,7 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
259 void save(float *ptr) const { _mm256_storeu_ps(ptr, reg); }
260 };
261
262 +#ifdef __AVX512F__
263 struct FP32Vec16 : public Vec<FP32Vec16> {
264 constexpr static int VEC_ELEM_NUM = 16;
265 union AliasReg {
@@ -290,6 +324,114 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
324
325 void save(float *ptr) const { _mm512_storeu_ps(ptr, reg); }
326 };
327 +#else
328 +struct FP32Vec16 : public Vec<FP32Vec16> {
329 + constexpr static int VEC_ELEM_NUM = 16;
330 +
331 + union AliasReg {
332 + __m256 reg;
333 + float values[8];
334 + };
335 +
336 + __m256 reg_low;
337 + __m256 reg_high;
338 +
339 + explicit FP32Vec16(float v) : reg_low(_mm256_set1_ps(v)),
340 + reg_high(_mm256_set1_ps(v)) {}
341 +
342 + explicit FP32Vec16() : reg_low(_mm256_set1_ps(0.0)),
343 + reg_high(_mm256_set1_ps(0.0)) {}
344 +
345 + explicit FP32Vec16(const float *ptr) : reg_low(_mm256_loadu_ps(ptr)),
346 + reg_high(_mm256_loadu_ps(ptr + 8)) {}
347 +
348 + explicit FP32Vec16(__m256 low, __m256 high) : reg_low(low), reg_high(high) {}
349 +
350 + explicit FP32Vec16(const FP32Vec16 &data) : reg_low(data.reg_low),
351 + reg_high(data.reg_high) {}
352 +
353 + explicit FP32Vec16(const FP32Vec4 &data)
354 + : reg_low((__m256)_mm256_inserti128_si256(
355 + _mm256_castsi128_si256((__m128i)data.reg),
356 + (__m128i)data.reg, 1)),
357 + reg_high((__m256)_mm256_inserti128_si256(
358 + _mm256_castsi128_si256((__m128i)data.reg),
359 + (__m128i)data.reg, 1)) {}
360 +
361 + explicit FP32Vec16(const FP32Vec8 &data)
362 + : reg_low(data.reg), reg_high(data.reg) {}
363 +
364 + explicit FP32Vec16(const BF16Vec16 &v) {
365 + __m128i low = _mm256_extractf128_si256(v.reg, 0);
366 + __m128i high = _mm256_extractf128_si256(v.reg, 1);
367 +
368 + __m256i v_low_epi32 = _mm256_cvtepu16_epi32(low);
369 + __m256i v_high_epi32 = _mm256_cvtepu16_epi32(high);
370 +
371 + __m256i v_low_shifted = _mm256_bslli_epi128(v_low_epi32, 2);
372 + __m256i v_high_shifted = _mm256_bslli_epi128(v_high_epi32, 2);
373 +
374 + reg_low = _mm256_castsi256_ps(v_low_shifted);
375 + reg_high = _mm256_castsi256_ps(v_high_shifted);
376 + }
377 +
378 + explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
379 +
380 + FP32Vec16 operator*(const FP32Vec16 &b) const {
381 + return FP32Vec16(_mm256_mul_ps(reg_low, b.reg_low),
382 + _mm256_mul_ps(reg_high, b.reg_high));
383 + }
384 +
385 + FP32Vec16 operator+(const FP32Vec16 &b) const {
386 + return FP32Vec16(_mm256_add_ps(reg_low, b.reg_low),
387 + _mm256_add_ps(reg_high, b.reg_high));
388 + }
389 +
390 + FP32Vec16 operator-(const FP32Vec16 &b) const {
391 + return FP32Vec16(_mm256_sub_ps(reg_low, b.reg_low),
392 + _mm256_sub_ps(reg_high, b.reg_high));
393 + }
394 +
395 + FP32Vec16 operator/(const FP32Vec16 &b) const {
396 + return FP32Vec16(_mm256_div_ps(reg_low, b.reg_low),
397 + _mm256_div_ps(reg_high, b.reg_high));
398 + }
399 +
400 + float reduce_sum() const {
401 + FP32Vec8 low = FP32Vec8(reg_low);
402 + FP32Vec8 high = FP32Vec8(reg_high);
403 + return low.reduce_sum() + high.reduce_sum();
404 + }
405 +
406 + template <int group_size> float reduce_sub_sum(int idx) {
407 + float sum = 0.0;
408 + static_assert(VEC_ELEM_NUM % group_size == 0);
409 + constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
410 + uint32_t mask = base_mask << (idx * group_size);
411 +
412 + AliasReg ar;
413 +
414 + auto func = [&sum, &mask, &ar](int i) {
415 + int flag = mask & 0x1;
416 + mask = mask >> 1;
417 + if (flag != 0) sum += ar.values[i];
418 + };
419 +
420 + ar.reg = reg_low;
421 + unroll_loop<int, 8>(func);
422 +
423 + ar.reg = reg_high;
424 + unroll_loop<int, 8>(func);
425 +
426 + return sum;
427 + }
428 +
429 + void save(float *ptr) const {
430 + _mm256_storeu_ps(ptr, reg_low);
431 + _mm256_storeu_ps(ptr + 8, reg_high);
432 + }
433 +};
434 +#endif
435
436 template <typename T> struct VecType { using vec_type = void; };
437
@@ -336,6 +478,7 @@ template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
478 *ptr = *(v_ptr + 1);
479 }
480
481 +#ifdef __AVX512F__
482 inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
483 : reg(_mm256_cvtepi32_epi16(
484 _mm256_bsrli_epi128(_mm256_castps_si256(v.reg), 2))) {}
@@ -343,7 +486,27 @@ inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
486 inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
487 : reg(_mm512_cvtepi32_epi16(
488 _mm512_bsrli_epi128(_mm512_castps_si512(v.reg), 2))) {}
346 -#endif
489 +#else
490 +namespace{
491 +__m128i FP32Vec8_to_BF16Vec8_avx2(__m256 a) {
492 + __m256i ai = _mm256_castps_si256(a);
493 + ai = _mm256_srli_epi32(ai, 16);
494 + ai = _mm256_packus_epi32(ai, ai);
495 + ai = _mm256_permute4x64_epi64(ai, 0b00111001);
496 + return _mm256_extracti128_si256(ai, 0);
497 +}
498 +}
499 +
500 +inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
501 + : reg(FP32Vec8_to_BF16Vec8_avx2(v.reg)) {}
502 +
503 +inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
504 + BF16Vec8 low = BF16Vec8(FP32Vec8(v.reg_low));
505 + BF16Vec8 high = BF16Vec8(FP32Vec8(v.reg_high));
506 + reg = _mm256_insertf128_si256(_mm256_castsi128_si256(low.reg), high.reg, 1);
507 +}
508 +#endif // __AVX512F__
509 +#endif // __AVX512BF16__
510
511 inline void prefetch(const void *addr) { _mm_prefetch(addr, _MM_HINT_T1); }
512