@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c81dddb45c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[ROCm] Fix build problem resulted from previous commit related to FP8 kv-cache support (#2790)

Hongxia Yang committed Feb 7, 2024 at 01:36 UTC c81dddb45c71e630b907f9d84686ecd73b4105c7
2 files changed +16
Dockerfile.rocm
+1
@@ -76,6 +76,7 @@ RUN cd /app \
76 && cd vllm \
77 && pip install -U -r requirements-rocm.txt \
78 && bash patch_xformers.rocm.sh \
79 + && patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h /app/vllm/rocm_patch/rocm_bf16.patch \
80 && python3 setup.py install \
81 && cd ..
82
rocm_patch/rocm_bf16.patch new
+15
@@ -0,0 +1,15 @@
1 +--- amd_hip_bf16.h 2024-02-06 18:28:58.268699142 +0000
2 ++++ amd_hip_bf16.h.new 2024-02-06 18:28:31.988647133 +0000
3 +@@ -90,10 +90,10 @@
4 + #include "math_fwd.h" // ocml device functions
5 +
6 + #if defined(__HIPCC_RTC__)
7 +-#define __HOST_DEVICE__ __device__
8 ++#define __HOST_DEVICE__ __device__ static
9 + #else
10 + #include <climits>
11 +-#define __HOST_DEVICE__ __host__ __device__
12 ++#define __HOST_DEVICE__ __host__ __device__ static inline
13 + #endif
14 +
15 + // Since we are using unsigned short to represent data in bfloat16, it can be of different sizes on