@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 64da65b322
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Prefix Caching- fix t4 triton error (#2517)

shiyi.c_98 committed Feb 16, 2024 at 14:17 UTC 64da65b3225b7a2e6c2b161b726dc9751f973f33
1 file changed +3 -1
vllm/model_executor/layers/triton_kernel/prefix_prefill.py
+3 -1
@@ -618,7 +618,9 @@ if triton.__version__ >= "2.1.0":
618 b_ctx_len,
619 max_input_len,
620 alibi_slopes=None):
621 - BLOCK = 128
621 +
622 + cap = torch.cuda.get_device_capability()
623 + BLOCK = 128 if cap[0] >= 8 else 64
624 # shape constraints
625 Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
626 assert Lq == Lk and Lk == Lv