A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[BugFix] Fix NaN errors in paged attention kernel (#936)
Woosuk Kwon committed
Sep 4, 2023 at 09:20 UTC
bf87484efac9812b5854266025f86bbb21894e4b
4 files changed
+32
-5
csrc/attention/attention_kernels.cu
+12
@@ -246,6 +246,8 @@ __global__ void single_query_cached_kv_attention_kernel(
246
accs[i] = 0.f;
247
}
248
249
+ scalar_t zero_value;
250
+ zero(zero_value);
251
for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
252
const int physical_block_number = block_table[block_idx];
253
const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
@@ -261,6 +263,16 @@ __global__ void single_query_cached_kv_attention_kernel(
263
if (row_idx < HEAD_SIZE) {
264
const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
265
V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
266
+ if (block_idx == num_blocks - 1) {
267
+ // NOTE(woosuk): When v_vec contains the tokens that are out of the context,
268
+ // we should explicitly zero out the values since they may contain NaNs.
269
+ // See https://github.com/vllm-project/vllm/issues/641#issuecomment-1682544472
270
+ scalar_t* v_vec_ptr = reinterpret_cast<scalar_t*>(&v_vec);
271
+#pragma unroll
272
+ for (int j = 0; j <= V_VEC_SIZE; j++) {
273
+ v_vec_ptr[j] = token_idx + j < context_len ? v_vec_ptr[j] : zero_value;
274
+ }
275
+ }
276
accs[i] += dot(logits_vec, v_vec);
277
}
278
}
csrc/attention/dtype_bfloat16.cuh
+10
@@ -420,4 +420,14 @@ inline __device__ void from_float(bf16_8_t& dst, Float8_ src) {
420
#endif
421
}
422
423
+// Zero-out a variable.
424
+inline __device__ void zero(__nv_bfloat16& dst) {
425
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
426
+ assert(false);
427
+#else
428
+ // Same as CUDART_ZERO_BF16 introduced in CUDA 12.2.
429
+ dst = __ushort_as_bfloat16((unsigned short)0x0000U);
430
+#endif
431
+}
432
+
433
} // namespace vllm
csrc/attention/dtype_float16.cuh
+5
-5
@@ -390,11 +390,6 @@ inline __device__ float sum(uint4 v) {
390
return sum(c);
391
}
392
393
-// Zero-out a vector.
394
-inline __device__ void zero(uint16_t& dst) {
395
- dst = uint16_t(0);
396
-}
397
-
393
// From float32 to float16.
394
inline __device__ void from_float(uint16_t& dst, float src) {
395
dst = float_to_half(src);
@@ -441,4 +436,9 @@ inline __device__ Float8_ to_float(uint4 u) {
436
return tmp;
437
}
438
439
+// Zero-out a variable.
440
+inline __device__ void zero(uint16_t& dst) {
441
+ dst = uint16_t(0);
442
+}
443
+
444
} // namespace vllm
csrc/attention/dtype_float32.cuh
+5
@@ -265,4 +265,9 @@ inline __device__ Float8_ to_float(Float8_ u) {
265
return u;
266
}
267
268
+// Zero-out a variable.
269
+inline __device__ void zero(float& dst) {
270
+ dst = 0.f;
271
+}
272
+
273
} // namespace vllm