A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
hotfix attn alibi wo head mapping (#496)
Co-authored-by: oliveryuan <oliveryuan@basemind.com>
Song committed
Jul 19, 2023 at 02:31 UTC
bda41c70ddb124134935a90a0d51304d2ac035e8
2 files changed
+3
tests/kernels/test_attention.py
+2
@@ -199,6 +199,7 @@ def run_single_query_cached_kv_attention(
199
]
200
block_tables.append(block_table)
201
block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
202
+ head_mapping = torch.arange(num_heads, dtype=torch.int32, device="cuda")
203
204
scale = float(1.0 / (head_size**0.5))
205
output = torch.empty(num_tokens,
@@ -211,6 +212,7 @@ def run_single_query_cached_kv_attention(
212
query,
213
key_cache,
214
value_cache,
215
+ head_mapping,
216
scale,
217
block_tables,
218
context_lens,
vllm/model_executor/layers/attention.py
+1
@@ -408,6 +408,7 @@ class PagedAttentionWithALiBi(PagedAttention):
408
query,
409
key_cache,
410
value_cache,
411
+ self.head_mapping,
412
self.scale,
413
input_metadata.block_tables,
414
input_metadata.context_lens,