@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ebe4d1db3a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix boundary check in paged attention kernel (#1241)

Liang committed Oct 2, 2023 at 02:35 UTC ebe4d1db3a42096cebcc2b2d289143bc0ef02d3d
1 file changed +1 -1
csrc/attention/attention_kernels.cu
+1 -1
@@ -269,7 +269,7 @@ __global__ void single_query_cached_kv_attention_kernel(
269 // See https://github.com/vllm-project/vllm/issues/641#issuecomment-1682544472
270 scalar_t* v_vec_ptr = reinterpret_cast<scalar_t*>(&v_vec);
271 #pragma unroll
272 - for (int j = 0; j <= V_VEC_SIZE; j++) {
272 + for (int j = 0; j < V_VEC_SIZE; j++) {
273 v_vec_ptr[j] = token_idx + j < context_len ? v_vec_ptr[j] : zero_value;
274 }
275 }