A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BugFix] Fix UB in per_token_group_quant.cu (#24913)
Signed-off-by: Shreeasish Kumar <shreeasish@rivosinc.com>
rivos-shreeasish committed
Sep 23, 2025 at 09:14 UTC
2357480b1abefb2b7177046133080917e1449324
1 file changed
+3
-3
csrc/quantization/fp8/per_token_group_quant.cu
+3
-3
@@ -12,8 +12,8 @@
12
#include "../vectorization_utils.cuh"
13
#include "../../dispatch_utils.h"
14
15
-__device__ __forceinline__ float GroupReduceMax(float val, const int tid) {
16
- unsigned mask = 0xffff;
15
+__device__ __forceinline__ float GroupReduceMax(float val) {
16
+ unsigned mask = threadIdx.x % 32 >= 16 ? 0xffff0000 : 0x0000ffff;
17
18
val = fmaxf(val, __shfl_xor_sync(mask, val, 8));
19
val = fmaxf(val, __shfl_xor_sync(mask, val, 4));
@@ -86,7 +86,7 @@ __global__ void per_token_group_quant_8bit_kernel(
86
threads_per_group, // stride in group
87
scalar_op_cache); // scalar handler
88
89
- local_absmax = GroupReduceMax(local_absmax, lane_id);
89
+ local_absmax = GroupReduceMax(local_absmax);
90
91
float y_s = local_absmax / max_8bit;
92
if constexpr (SCALE_UE8M0) {