@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a6f332d0d9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Hardware][CPU][bugfix] Fix half dtype support on AVX2-only target (#10108)

Signed-off-by: jiang1.li <jiang1.li@intel.com>

Li, Jiang committed Nov 7, 2024 at 18:42 UTC a6f332d0d9ac3e795949da7703f203b6b1a42797
2 files changed +11 -1
cmake/cpu_extension.cmake
+1 -1
@@ -93,7 +93,7 @@ if (AVX512_FOUND AND NOT AVX512_DISABLED)
93 FetchContent_Declare(
94 oneDNN
95 GIT_REPOSITORY https://github.com/oneapi-src/oneDNN.git
96 - GIT_TAG v3.5.3
96 + GIT_TAG v3.6
97 GIT_PROGRESS TRUE
98 GIT_SHALLOW TRUE
99 )
csrc/cpu/cpu_types_x86.hpp
+10
@@ -432,6 +432,16 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
432 explicit FP32Vec16(const FP32Vec8 &data)
433 : reg_low(data.reg), reg_high(data.reg) {}
434
435 + explicit FP32Vec16(const FP16Vec16 &v) {
436 + __m128i low = _mm256_extractf128_si256(v.reg, 0);
437 + __m128i high = _mm256_extractf128_si256(v.reg, 1);
438 +
439 + reg_low = _mm256_cvtph_ps(low);
440 + reg_high = _mm256_cvtph_ps(high);
441 + }
442 +
443 + explicit FP32Vec16(const FP16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
444 +
445 explicit FP32Vec16(const BF16Vec16 &v) {
446 __m128i low = _mm256_extractf128_si256(v.reg, 0);
447 __m128i high = _mm256_extractf128_si256(v.reg, 1);