A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] Fix hard-coded value of x in context_attention_fwd (#6373)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Thomas Parnell committed
Jul 13, 2024 at 03:30 UTC
e1684a766ad3f2f1531c273fa8a056fc14c4c71e
1 file changed
+2
-2
vllm/attention/ops/prefix_prefill.py
+2
-2
@@ -718,7 +718,7 @@ if triton.__version__ >= "2.1.0":
718
b_ctx_len,
719
alibi_slopes,
720
v_cache.shape[3],
721
- 8,
721
+ k_cache.shape[4],
722
o,
723
b_loc.stride(0),
724
b_loc.stride(1),
@@ -768,7 +768,7 @@ if triton.__version__ >= "2.1.0":
768
b_seq_len,
769
b_ctx_len,
770
v_cache.shape[3],
771
- 8,
771
+ k_cache.shape[4],
772
o,
773
b_loc.stride(0),
774
b_loc.stride(1),