A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix a bug in attention kernel (#68)
Woosuk Kwon committed
May 4, 2023 at 02:56 UTC
130d5fd8c7902a5a2e54feb25249168151e3bc2f
1 file changed
+1
-1
csrc/attention/attention_kernels.cu
+1
-1
@@ -345,7 +345,7 @@ void single_query_cached_kv_attention_launcher(
345
346
constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
347
int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
348
- int logits_size = padded_max_context_len * sizeof(T);
348
+ int logits_size = padded_max_context_len * sizeof(float);
349
int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
350
int shared_mem_size = std::max(logits_size, outputs_size);
351