A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix][Kernel] Prevent integer overflow in fp8 dynamic per-token quantize kernel (#9425)
Tyler Michael Smith committed
Oct 16, 2024 at 19:46 UTC
c3fab5f7691c55e9fd0de5ed373f4dd5fb2152cf
1 file changed
+4
-2
csrc/quantization/fp8/common.cu
+4
-2
@@ -204,8 +204,10 @@ __global__ void dynamic_per_token_scaled_fp8_quant_kernel(
204
int const tid = threadIdx.x;
205
int const token_idx = blockIdx.x;
206
207
- scalar_t const* __restrict__ token_input = &input[token_idx * hidden_size];
208
- FP8_TYPE* __restrict__ token_output = &out[token_idx * hidden_size];
207
+ // Use int64 to avoid overflowing an int32 when calculating this offset
208
+ int64_t offset = static_cast<int64_t>(token_idx) * hidden_size;
209
+ scalar_t const* __restrict__ token_input = &input[offset];
210
+ FP8_TYPE* __restrict__ token_output = &out[offset];
211
212
// For vectorization, token_input and token_output pointers need to be
213
// aligned at 8-byte and 4-byte addresses respectively.