@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bda41c70dd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

hotfix attn alibi wo head mapping (#496)

Co-authored-by: oliveryuan <oliveryuan@basemind.com>

Song committed Jul 19, 2023 at 02:31 UTC bda41c70ddb124134935a90a0d51304d2ac035e8
2 files changed +3
tests/kernels/test_attention.py
+2
@@ -199,6 +199,7 @@ def run_single_query_cached_kv_attention(
199 ]
200 block_tables.append(block_table)
201 block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
202 + head_mapping = torch.arange(num_heads, dtype=torch.int32, device="cuda")
203
204 scale = float(1.0 / (head_size**0.5))
205 output = torch.empty(num_tokens,
@@ -211,6 +212,7 @@ def run_single_query_cached_kv_attention(
212 query,
213 key_cache,
214 value_cache,
215 + head_mapping,
216 scale,
217 block_tables,
218 context_lens,
vllm/model_executor/layers/attention.py
+1
@@ -408,6 +408,7 @@ class PagedAttentionWithALiBi(PagedAttention):
408 query,
409 key_cache,
410 value_cache,
411 + self.head_mapping,
412 self.scale,
413 input_metadata.block_tables,
414 input_metadata.context_lens,