@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 819b18e7ba
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Rewrite torch.repeat_interleave to remove cpu synchronization (#1599)

ljss committed Nov 21, 2023 at 09:46 UTC 819b18e7ba7f179ba90e44b2a846ddbdd1b0763d
1 file changed +20 -10
vllm/model_executor/layers/attention.py
+20 -10
@@ -95,10 +95,15 @@ class PagedAttention(nn.Module):
95 """
96 if self.num_kv_heads != self.num_heads:
97 # Project the key and value tensors to the desired number of heads.
98 - key = torch.repeat_interleave(key, self.num_queries_per_kv, dim=1)
99 - value = torch.repeat_interleave(value,
100 - self.num_queries_per_kv,
101 - dim=1)
98 + query = query.view(query.shape[0], self.num_kv_heads,
99 + self.num_queries_per_kv, query.shape[-1])
100 + key = key[:, :,
101 + None, :].expand(key.shape[0], self.num_kv_heads,
102 + self.num_queries_per_kv, key.shape[-1])
103 + value = value[:, :,
104 + None, :].expand(value.shape[0], self.num_kv_heads,
105 + self.num_queries_per_kv,
106 + value.shape[-1])
107
108 # TODO(woosuk): The unsqueeze op may incur some CPU overhead. Optimize.
109 out = xops.memory_efficient_attention_forward(
@@ -110,7 +115,7 @@ class PagedAttention(nn.Module):
115 scale=self.scale,
116 )
117 # TODO(woosuk): Unnecessary copy. Optimize.
113 - output.copy_(out.squeeze(0))
118 + output.copy_(out.view_as(output))
119 return output
120
121 def get_alibi_slopes(self) -> Optional[torch.Tensor]:
@@ -427,10 +432,15 @@ class PagedAttentionWithALiBi(PagedAttention):
432 """
433 if self.num_kv_heads != self.num_heads:
434 # Project the key and value tensors to the desired number of heads.
430 - key = torch.repeat_interleave(key, self.num_queries_per_kv, dim=1)
431 - value = torch.repeat_interleave(value,
432 - self.num_queries_per_kv,
433 - dim=1)
435 + query = query.view(query.shape[0], self.num_kv_heads,
436 + self.num_queries_per_kv, query.shape[-1])
437 + key = key[:, :,
438 + None, :].expand(key.shape[0], self.num_kv_heads,
439 + self.num_queries_per_kv, key.shape[-1])
440 + value = value[:, :,
441 + None, :].expand(value.shape[0], self.num_kv_heads,
442 + self.num_queries_per_kv,
443 + value.shape[-1])
444 batch_size = input_metadata.num_prompts
445 seq_len = input_metadata.max_prompt_len
446
@@ -443,7 +453,7 @@ class PagedAttentionWithALiBi(PagedAttention):
453 scale=self.scale,
454 )
455 # TODO(woosuk): Unnecessary copy. Optimize.
446 - output.copy_(out.view(-1, self.num_heads, self.head_size))
456 + output.copy_(out.view_as(output))
457 return output
458
459 def get_alibi_slopes(self) -> Optional[torch.Tensor]: