A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix boundary check in paged attention kernel (#1241)
Liang committed
Oct 2, 2023 at 02:35 UTC
ebe4d1db3a42096cebcc2b2d289143bc0ef02d3d
1 file changed
+1
-1
csrc/attention/attention_kernels.cu
+1
-1
@@ -269,7 +269,7 @@ __global__ void single_query_cached_kv_attention_kernel(
269
// See https://github.com/vllm-project/vllm/issues/641#issuecomment-1682544472
270
scalar_t* v_vec_ptr = reinterpret_cast<scalar_t*>(&v_vec);
271
#pragma unroll
272
- for (int j = 0; j <= V_VEC_SIZE; j++) {
272
+ for (int j = 0; j < V_VEC_SIZE; j++) {
273
v_vec_ptr[j] = token_idx + j < context_len ? v_vec_ptr[j] : zero_value;
274
}
275
}