@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 762fd1c3fa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Refactor and annotate types for attention

Woosuk Kwon committed Feb 24, 2023 at 08:58 UTC 762fd1c3faf17001b943cb68e5d08e7d7fe59119
1 file changed +40 -32
cacheflow/models/attention.py
+40 -32
@@ -1,4 +1,4 @@
1 -from typing import Optional
1 +from typing import List, Optional
2
3 import torch
4 import torch.nn as nn
@@ -30,24 +30,34 @@ class OPTCacheFlowAttention(nn.Module):
30
31 def multi_query_kv_attention(
32 self,
33 - output: torch.Tensor,
34 - query: torch.Tensor,
35 - key: torch.Tensor,
36 - value: torch.Tensor,
33 + output: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
34 + query: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
35 + key: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
36 + value: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
37 + prompt_lens: List[int],
38 ) -> None:
38 - # FIXME(woosuk): Replace this with a custom op call.
39 - attention_mask = torch.triu(
40 - torch.ones(query.shape[0], key.shape[0]), diagonal=1) * -1e5
41 - attention_mask = attention_mask.to(dtype=query.dtype, device=query.device)
42 - out = self._masked_attention(query, key, value, attention_mask)
43 - output.copy_(out, non_blocking=True)
39 + # FIXME(woosuk): Replace the following with a custom op.
40 + start_idx = 0
41 + for prompt_len in prompt_lens:
42 + out = output[start_idx:start_idx + prompt_len]
43 + q = query[start_idx:start_idx + prompt_len]
44 + k = key[start_idx:start_idx + prompt_len]
45 + v = value[start_idx:start_idx + prompt_len]
46 +
47 + attention_mask = torch.triu(
48 + torch.ones(q.shape[0], k.shape[0]), diagonal=1) * -1e5
49 + attention_mask = attention_mask.to(dtype=q.dtype, device=q.device)
50 + attention_out = self._masked_attention(q, k, v, attention_mask)
51 + out.copy_(attention_out, non_blocking=True)
52 +
53 + start_idx += prompt_len
54
55 def single_query_cached_kv_attention(
56 self,
47 - output: torch.Tensor,
48 - query: torch.Tensor,
49 - key_cache: torch.Tensor,
50 - value_cache: torch.Tensor,
57 + output: torch.Tensor, # [num_generation_tokens, num_heads, head_size]
58 + query: torch.Tensor, # [num_generation_tokens, num_heads, head_size]
59 + key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
60 + value_cache: torch.Tensor, # [num_blocks, num_heads, block_size, head_size]
61 input_metadata: InputMetadata,
62 ) -> None:
63 num_heads = value_cache.shape[1]
@@ -82,15 +92,18 @@ class OPTCacheFlowAttention(nn.Module):
92
93 def forward(
94 self,
85 - query: torch.Tensor,
86 - key: torch.Tensor,
87 - value: torch.Tensor,
88 - key_cache: torch.Tensor,
89 - value_cache: torch.Tensor,
95 + query: torch.Tensor, # [num_tokens, num_heads * head_size]
96 + key: torch.Tensor, # [num_tokens, num_heads * head_size]
97 + value: torch.Tensor, # [num_tokens, num_heads * head_size]
98 + key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
99 + value_cache: torch.Tensor, # [num_blocks, num_heads, block_size, head_size]
100 input_metadata: InputMetadata,
101 cache_event: Optional[torch.cuda.Event],
92 - ) -> torch.Tensor:
93 - # Prune out invalid tokens.
102 + ) -> torch.Tensor: # [num_tokens, num_heads * head_size]
103 + # Pre-allocate the output tensor.
104 + output = torch.empty_like(query)
105 +
106 + # Prune out paddings if any.
107 query = query[:input_metadata.num_valid_tokens]
108 key = key[:input_metadata.num_valid_tokens]
109 value = value[:input_metadata.num_valid_tokens]
@@ -101,18 +114,11 @@ class OPTCacheFlowAttention(nn.Module):
114 query = query.view(-1, num_heads, head_size)
115 key = key.view(-1, num_heads, head_size)
116 value = value.view(-1, num_heads, head_size)
117 + output = output.view(-1, num_heads, head_size)
118
119 # Compute the attention op for prompts.
106 - output = torch.empty_like(query)
107 - start_idx = 0
108 - for i in range(input_metadata.num_prompts):
109 - prompt_len = input_metadata.prompt_lens[i]
110 - out = output[start_idx:start_idx + prompt_len]
111 - q = query[start_idx:start_idx + prompt_len]
112 - k = key[start_idx:start_idx + prompt_len]
113 - v = value[start_idx:start_idx + prompt_len]
114 - self.multi_query_kv_attention(out, q, k, v)
115 - start_idx += prompt_len
120 + self.multi_query_kv_attention(
121 + output, query, key, value, input_metadata.prompt_lens)
122
123 # Wait until the cache op is done.
124 if cache_event is not None:
@@ -124,6 +130,7 @@ class OPTCacheFlowAttention(nn.Module):
130
131 if input_metadata.num_generation_tokens > 0:
132 # Compute the attention op for generation tokens.
133 + start_idx = sum(input_metadata.prompt_lens)
134 self.single_query_cached_kv_attention(
135 output[start_idx:],
136 query[start_idx:],
@@ -132,4 +139,5 @@ class OPTCacheFlowAttention(nn.Module):
139 input_metadata)
140
141 # Reshape the output tensor.
142 + # NOTE(woosuk): The output tensor may include paddings.
143 return output.view(-1, num_heads * head_size)