A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
Fix triton compilation issue (#3984)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Bellk17 committed
Apr 12, 2024 at 16:41 UTC
d04973ad5446fe05c06035f6b2d99402fc3ac7bf
1 file changed
+5
-1
vllm/attention/ops/triton_flash_attention.py
+5
-1
@@ -415,7 +415,11 @@ def attn_fwd(
415
return
416
417
is_mqa = hq != hk
418
- off_h_k = off_h_q % hk if is_mqa else off_h_q
418
+ if is_mqa: # noqa: SIM108
419
+ off_h_k = off_h_q % hk
420
+ else:
421
+ off_h_k = off_h_q
422
+
423
n_extra_tokens = 0
424
if seqlen_k < BLOCK_N:
425
n_extra_tokens = BLOCK_N - seqlen_k