@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e4a28e5316
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[ROCM] Fix blockReduceSum to use correct warp counts for ROCm and CUDA (#3262)

Douglas Lehr committed Mar 10, 2024 at 17:27 UTC e4a28e53165902ffc5daf20977c70885d0c05768
3 files changed +13 -11
csrc/attention/attention_kernels.cu
-8
@@ -15,9 +15,6 @@
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18 -#ifdef USE_ROCM
19 -#include <hip/hip_runtime.h>
20 -#endif
18
19 #include <torch/extension.h>
20 #include <ATen/cuda/CUDAContext.h>
@@ -31,11 +28,6 @@
28
29 #include <algorithm>
30
34 -#ifndef USE_ROCM
35 -#define WARP_SIZE 32
36 -#else
37 -#define WARP_SIZE warpSize
38 -#endif
31 #define MAX(a, b) ((a) > (b) ? (a) : (b))
32 #define MIN(a, b) ((a) < (b) ? (a) : (b))
33 #define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
csrc/cuda_compat.h
+10
@@ -1,5 +1,15 @@
1 #pragma once
2
3 +#ifdef USE_ROCM
4 +#include <hip/hip_runtime.h>
5 +#endif
6 +
7 +#ifndef USE_ROCM
8 + #define WARP_SIZE 32
9 +#else
10 + #define WARP_SIZE warpSize
11 +#endif
12 +
13 #ifndef USE_ROCM
14 #define VLLM_LDG(arg) __ldg(arg)
15 #else
csrc/reduction_utils.cuh
+3 -3
@@ -24,7 +24,7 @@ namespace vllm {
24 template<typename T>
25 __inline__ __device__ T warpReduceSum(T val) {
26 #pragma unroll
27 - for (int mask = 16; mask > 0; mask >>= 1)
27 + for (int mask = WARP_SIZE/2; mask > 0; mask >>= 1)
28 val += VLLM_SHFL_XOR_SYNC(val, mask);
29 return val;
30 }
@@ -32,7 +32,7 @@ __inline__ __device__ T warpReduceSum(T val) {
32 /* Calculate the sum of all elements in a block */
33 template<typename T>
34 __inline__ __device__ T blockReduceSum(T val) {
35 - static __shared__ T shared[32];
35 + static __shared__ T shared[WARP_SIZE];
36 int lane = threadIdx.x & 0x1f;
37 int wid = threadIdx.x >> 5;
38
@@ -45,7 +45,7 @@ __inline__ __device__ T blockReduceSum(T val) {
45
46 // Modify from blockDim.x << 5 to blockDim.x / 32. to prevent
47 // blockDim.x is not divided by 32
48 - val = (threadIdx.x < (blockDim.x / 32.f)) ? shared[lane] : (T)(0.0f);
48 + val = (threadIdx.x < (blockDim.x / (WARP_SIZE * 1.0f))) ? shared[lane] : (T)(0.0f);
49 val = warpReduceSum<T>(val);
50 return val;
51 }