@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c3fab5f769
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix][Kernel] Prevent integer overflow in fp8 dynamic per-token quantize kernel (#9425)

Tyler Michael Smith committed Oct 16, 2024 at 19:46 UTC c3fab5f7691c55e9fd0de5ed373f4dd5fb2152cf
1 file changed +4 -2
csrc/quantization/fp8/common.cu
+4 -2
@@ -204,8 +204,10 @@ __global__ void dynamic_per_token_scaled_fp8_quant_kernel(
204 int const tid = threadIdx.x;
205 int const token_idx = blockIdx.x;
206
207 - scalar_t const* __restrict__ token_input = &input[token_idx * hidden_size];
208 - FP8_TYPE* __restrict__ token_output = &out[token_idx * hidden_size];
207 + // Use int64 to avoid overflowing an int32 when calculating this offset
208 + int64_t offset = static_cast<int64_t>(token_idx) * hidden_size;
209 + scalar_t const* __restrict__ token_input = &input[offset];
210 + FP8_TYPE* __restrict__ token_output = &out[offset];
211
212 // For vectorization, token_input and token_output pointers need to be
213 // aligned at 8-byte and 4-byte addresses respectively.