A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[ROCM] Fix blockReduceSum to use correct warp counts for ROCm and CUDA (#3262)
Douglas Lehr committed
Mar 10, 2024 at 17:27 UTC
e4a28e53165902ffc5daf20977c70885d0c05768
3 files changed
+13
-11
csrc/attention/attention_kernels.cu
-8
@@ -15,9 +15,6 @@
15
* See the License for the specific language governing permissions and
16
* limitations under the License.
17
*/
18
-#ifdef USE_ROCM
19
-#include <hip/hip_runtime.h>
20
-#endif
18
19
#include <torch/extension.h>
20
#include <ATen/cuda/CUDAContext.h>
@@ -31,11 +28,6 @@
28
29
#include <algorithm>
30
34
-#ifndef USE_ROCM
35
-#define WARP_SIZE 32
36
-#else
37
-#define WARP_SIZE warpSize
38
-#endif
31
#define MAX(a, b) ((a) > (b) ? (a) : (b))
32
#define MIN(a, b) ((a) < (b) ? (a) : (b))
33
#define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
csrc/cuda_compat.h
+10
@@ -1,5 +1,15 @@
1
#pragma once
2
3
+#ifdef USE_ROCM
4
+#include <hip/hip_runtime.h>
5
+#endif
6
+
7
+#ifndef USE_ROCM
8
+ #define WARP_SIZE 32
9
+#else
10
+ #define WARP_SIZE warpSize
11
+#endif
12
+
13
#ifndef USE_ROCM
14
#define VLLM_LDG(arg) __ldg(arg)
15
#else
csrc/reduction_utils.cuh
+3
-3
@@ -24,7 +24,7 @@ namespace vllm {
24
template<typename T>
25
__inline__ __device__ T warpReduceSum(T val) {
26
#pragma unroll
27
- for (int mask = 16; mask > 0; mask >>= 1)
27
+ for (int mask = WARP_SIZE/2; mask > 0; mask >>= 1)
28
val += VLLM_SHFL_XOR_SYNC(val, mask);
29
return val;
30
}
@@ -32,7 +32,7 @@ __inline__ __device__ T warpReduceSum(T val) {
32
/* Calculate the sum of all elements in a block */
33
template<typename T>
34
__inline__ __device__ T blockReduceSum(T val) {
35
- static __shared__ T shared[32];
35
+ static __shared__ T shared[WARP_SIZE];
36
int lane = threadIdx.x & 0x1f;
37
int wid = threadIdx.x >> 5;
38
@@ -45,7 +45,7 @@ __inline__ __device__ T blockReduceSum(T val) {
45
46
// Modify from blockDim.x << 5 to blockDim.x / 32. to prevent
47
// blockDim.x is not divided by 32
48
- val = (threadIdx.x < (blockDim.x / 32.f)) ? shared[lane] : (T)(0.0f);
48
+ val = (threadIdx.x < (blockDim.x / (WARP_SIZE * 1.0f))) ? shared[lane] : (T)(0.0f);
49
val = warpReduceSum<T>(val);
50
return val;
51
}