A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Rewrite torch.repeat_interleave to remove cpu synchronization (#1599)
ljss committed
Nov 21, 2023 at 09:46 UTC
819b18e7ba7f179ba90e44b2a846ddbdd1b0763d
1 file changed
+20
-10
vllm/model_executor/layers/attention.py
+20
-10
@@ -95,10 +95,15 @@ class PagedAttention(nn.Module):
95
"""
96
if self.num_kv_heads != self.num_heads:
97
# Project the key and value tensors to the desired number of heads.
98
- key = torch.repeat_interleave(key, self.num_queries_per_kv, dim=1)
99
- value = torch.repeat_interleave(value,
100
- self.num_queries_per_kv,
101
- dim=1)
98
+ query = query.view(query.shape[0], self.num_kv_heads,
99
+ self.num_queries_per_kv, query.shape[-1])
100
+ key = key[:, :,
101
+ None, :].expand(key.shape[0], self.num_kv_heads,
102
+ self.num_queries_per_kv, key.shape[-1])
103
+ value = value[:, :,
104
+ None, :].expand(value.shape[0], self.num_kv_heads,
105
+ self.num_queries_per_kv,
106
+ value.shape[-1])
107
108
# TODO(woosuk): The unsqueeze op may incur some CPU overhead. Optimize.
109
out = xops.memory_efficient_attention_forward(
@@ -110,7 +115,7 @@ class PagedAttention(nn.Module):
115
scale=self.scale,
116
)
117
# TODO(woosuk): Unnecessary copy. Optimize.
113
- output.copy_(out.squeeze(0))
118
+ output.copy_(out.view_as(output))
119
return output
120
121
def get_alibi_slopes(self) -> Optional[torch.Tensor]:
@@ -427,10 +432,15 @@ class PagedAttentionWithALiBi(PagedAttention):
432
"""
433
if self.num_kv_heads != self.num_heads:
434
# Project the key and value tensors to the desired number of heads.
430
- key = torch.repeat_interleave(key, self.num_queries_per_kv, dim=1)
431
- value = torch.repeat_interleave(value,
432
- self.num_queries_per_kv,
433
- dim=1)
435
+ query = query.view(query.shape[0], self.num_kv_heads,
436
+ self.num_queries_per_kv, query.shape[-1])
437
+ key = key[:, :,
438
+ None, :].expand(key.shape[0], self.num_kv_heads,
439
+ self.num_queries_per_kv, key.shape[-1])
440
+ value = value[:, :,
441
+ None, :].expand(value.shape[0], self.num_kv_heads,
442
+ self.num_queries_per_kv,
443
+ value.shape[-1])
444
batch_size = input_metadata.num_prompts
445
seq_len = input_metadata.max_prompt_len
446
@@ -443,7 +453,7 @@ class PagedAttentionWithALiBi(PagedAttention):
453
scale=self.scale,
454
)
455
# TODO(woosuk): Unnecessary copy. Optimize.
446
- output.copy_(out.view(-1, self.num_heads, self.head_size))
456
+ output.copy_(out.view_as(output))
457
return output
458
459
def get_alibi_slopes(self) -> Optional[torch.Tensor]: