@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d04973ad54
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

Fix triton compilation issue (#3984)

Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Bellk17 committed Apr 12, 2024 at 16:41 UTC d04973ad5446fe05c06035f6b2d99402fc3ac7bf
1 file changed +5 -1
vllm/attention/ops/triton_flash_attention.py
+5 -1
@@ -415,7 +415,11 @@ def attn_fwd(
415 return
416
417 is_mqa = hq != hk
418 - off_h_k = off_h_q % hk if is_mqa else off_h_q
418 + if is_mqa: # noqa: SIM108
419 + off_h_k = off_h_q % hk
420 + else:
421 + off_h_k = off_h_q
422 +
423 n_extra_tokens = 0
424 if seqlen_k < BLOCK_N:
425 n_extra_tokens = BLOCK_N - seqlen_k