@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bf87484efa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[BugFix] Fix NaN errors in paged attention kernel (#936)

Woosuk Kwon committed Sep 4, 2023 at 09:20 UTC bf87484efac9812b5854266025f86bbb21894e4b
4 files changed +32 -5
csrc/attention/attention_kernels.cu
+12
@@ -246,6 +246,8 @@ __global__ void single_query_cached_kv_attention_kernel(
246 accs[i] = 0.f;
247 }
248
249 + scalar_t zero_value;
250 + zero(zero_value);
251 for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
252 const int physical_block_number = block_table[block_idx];
253 const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
@@ -261,6 +263,16 @@ __global__ void single_query_cached_kv_attention_kernel(
263 if (row_idx < HEAD_SIZE) {
264 const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
265 V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
266 + if (block_idx == num_blocks - 1) {
267 + // NOTE(woosuk): When v_vec contains the tokens that are out of the context,
268 + // we should explicitly zero out the values since they may contain NaNs.
269 + // See https://github.com/vllm-project/vllm/issues/641#issuecomment-1682544472
270 + scalar_t* v_vec_ptr = reinterpret_cast<scalar_t*>(&v_vec);
271 +#pragma unroll
272 + for (int j = 0; j <= V_VEC_SIZE; j++) {
273 + v_vec_ptr[j] = token_idx + j < context_len ? v_vec_ptr[j] : zero_value;
274 + }
275 + }
276 accs[i] += dot(logits_vec, v_vec);
277 }
278 }
csrc/attention/dtype_bfloat16.cuh
+10
@@ -420,4 +420,14 @@ inline __device__ void from_float(bf16_8_t& dst, Float8_ src) {
420 #endif
421 }
422
423 +// Zero-out a variable.
424 +inline __device__ void zero(__nv_bfloat16& dst) {
425 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
426 + assert(false);
427 +#else
428 + // Same as CUDART_ZERO_BF16 introduced in CUDA 12.2.
429 + dst = __ushort_as_bfloat16((unsigned short)0x0000U);
430 +#endif
431 +}
432 +
433 } // namespace vllm
csrc/attention/dtype_float16.cuh
+5 -5
@@ -390,11 +390,6 @@ inline __device__ float sum(uint4 v) {
390 return sum(c);
391 }
392
393 -// Zero-out a vector.
394 -inline __device__ void zero(uint16_t& dst) {
395 - dst = uint16_t(0);
396 -}
397 -
393 // From float32 to float16.
394 inline __device__ void from_float(uint16_t& dst, float src) {
395 dst = float_to_half(src);
@@ -441,4 +436,9 @@ inline __device__ Float8_ to_float(uint4 u) {
436 return tmp;
437 }
438
439 +// Zero-out a variable.
440 +inline __device__ void zero(uint16_t& dst) {
441 + dst = uint16_t(0);
442 +}
443 +
444 } // namespace vllm
csrc/attention/dtype_float32.cuh
+5
@@ -265,4 +265,9 @@ inline __device__ Float8_ to_float(Float8_ u) {
265 return u;
266 }
267
268 +// Zero-out a variable.
269 +inline __device__ void zero(float& dst) {
270 + dst = 0.f;
271 +}
272 +
273 } // namespace vllm