A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Prefix Caching- fix t4 triton error (#2517)
shiyi.c_98 committed
Feb 16, 2024 at 14:17 UTC
64da65b3225b7a2e6c2b161b726dc9751f973f33
1 file changed
+3
-1
vllm/model_executor/layers/triton_kernel/prefix_prefill.py
+3
-1
@@ -618,7 +618,9 @@ if triton.__version__ >= "2.1.0":
618
b_ctx_len,
619
max_input_len,
620
alibi_slopes=None):
621
- BLOCK = 128
621
+
622
+ cap = torch.cuda.get_device_capability()
623
+ BLOCK = 128 if cap[0] >= 8 else 64
624
# shape constraints
625
Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
626
assert Lq == Lk and Lk == Lv