A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
Use the optimized block sizes after tuning the kernel. (#14329)
iefgnoix committed
Mar 7, 2025 at 05:25 UTC
1e3598edeb694c1a6a1fdfac5e7d12697749561b
1 file changed
+2
-2
vllm/v1/attention/backends/pallas.py
+2
-2
@@ -12,8 +12,8 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
12
from vllm.attention.backends.utils import CommonAttentionState
13
14
# These are the 2 tunable parameters of the paged attention Pallas kernel.
15
-NUM_QUERIES_PER_BLOCK = 32
16
-NUM_KV_PAGES_PER_BLOCK = 128
15
+NUM_QUERIES_PER_BLOCK = 16
16
+NUM_KV_PAGES_PER_BLOCK = 256
17
18
19
class PallasAttentionBackend(AttentionBackend):