@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1ea291a417
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

Fix: Build error seen on Power Architecture (#10421)

Signed-off-by: Manjul Mohan <manjul.mohan@ibm.com> Signed-off-by: B-201 <Joy25810@foxmail.com> Signed-off-by: Isotr0py <2037008807@qq.com> Signed-off-by: youkaichao <youkaichao@gmail.com> Signed-off-by: ismael-dm <ismaeldm99@gmail.com> Signed-off-by: Andrew Nesbitt <andrewnez@gmail.com> Signed-off-by: mgoin <michael@neuralmagic.com> Signed-off-by: yan ma <yan.ma@intel.com> Signed-off-by: Angus Wang <wangjadehao@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: rickyx <rickyx@anyscale.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: Mengqing Cao <cmq0113@163.com> Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com> Co-authored-by: Manjul Mohan manjul.mohan@ibm.com <manjulmohan@ltcd97-lp2.aus.stglabs.ibm.com> Co-authored-by: B-201 <Joy25810@foxmail.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: youkaichao <youkaichao@gmail.com> Co-authored-by: ismael-dm <ismaeldm99@gmail.com> Co-authored-by: Andrew Nesbitt <andrewnez@gmail.com> Co-authored-by: Michael Goin <michael@neuralmagic.com> Co-authored-by: Yan Ma <yan.ma@intel.com> Co-authored-by: Angus Wang <wangjadehao@gmail.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Ricky Xu <rickyx@anyscale.com> Co-authored-by: Kevin H. Luu <kevin@anyscale.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Mengqing Cao <cmq0113@163.com> Co-authored-by: Travis Johnson <tsjohnso@us.ibm.com> Co-authored-by: Russell Bryant <rbryant@redhat.com>

Manjul Mohan committed Nov 19, 2024 at 23:04 UTC 1ea291a4173a82c537ab42487e23375be4926d30
3 files changed +26 -6
cmake/cpu_extension.cmake
+10 -4
@@ -16,10 +16,16 @@ include_directories("${CMAKE_SOURCE_DIR}/csrc")
16 #
17 # Check the compile flags
18 #
19 -list(APPEND CXX_COMPILE_FLAGS
20 - "-fopenmp"
21 - "-mf16c"
22 - "-DVLLM_CPU_EXTENSION")
19 +if (CMAKE_SYSTEM_PROCESSOR STREQUAL "ppc64le")
20 + list(APPEND CXX_COMPILE_FLAGS
21 + "-fopenmp"
22 + "-DVLLM_CPU_EXTENSION")
23 +else()
24 + list(APPEND CXX_COMPILE_FLAGS
25 + "-fopenmp"
26 + "-mf16c"
27 + "-DVLLM_CPU_EXTENSION")
28 +endif()
29
30 execute_process(COMMAND cat /proc/cpuinfo
31 RESULT_VARIABLE CPUINFO_RET
csrc/cpu/attention.cpp
+10 -2
@@ -24,12 +24,20 @@ struct KernelVecType<float> {
24
25 template <>
26 struct KernelVecType<c10::Half> {
27 +#ifdef __powerpc64__
28 + // Power architecture-specific vector types
29 + using q_load_vec_type = vec_op::FP32Vec8;
30 + using k_load_vec_type = vec_op::FP32Vec16;
31 + using v_load_vec_type = vec_op::FP32Vec16;
32 +#else
33 + // Fallback for other architectures, including x86
34 using q_load_vec_type = vec_op::FP16Vec8;
28 - using q_vec_type = vec_op::FP32Vec16;
35 using k_load_vec_type = vec_op::FP16Vec16;
36 + using v_load_vec_type = vec_op::FP16Vec16;
37 +#endif
38 + using q_vec_type = vec_op::FP32Vec16;
39 using k_vec_type = vec_op::FP32Vec16;
40 using qk_acc_vec_type = vec_op::FP32Vec16;
32 - using v_load_vec_type = vec_op::FP16Vec16;
41 };
42
43 #ifdef __AVX512BF16__
csrc/cpu/quant.cpp
+6
@@ -25,7 +25,13 @@ struct KernelVecType<c10::BFloat16> {
25
26 template <>
27 struct KernelVecType<c10::Half> {
28 +#ifdef __powerpc64__
29 + // Power architecture-specific vector type
30 + using load_vec_type = vec_op::FP32Vec16;
31 +#else
32 + // Fallback for other architectures
33 using load_vec_type = vec_op::FP16Vec16;
34 +#endif
35 using azp_adj_load_vec_type = vec_op::INT32Vec16;
36 using cvt_vec_type = vec_op::FP32Vec16;
37 };