A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Refactor and annotate types for attention
Woosuk Kwon committed
Feb 24, 2023 at 08:58 UTC
762fd1c3faf17001b943cb68e5d08e7d7fe59119
1 file changed
+40
-32
cacheflow/models/attention.py
+40
-32
@@ -1,4 +1,4 @@
1
-from typing import Optional
1
+from typing import List, Optional
2
3
import torch
4
import torch.nn as nn
@@ -30,24 +30,34 @@ class OPTCacheFlowAttention(nn.Module):
30
31
def multi_query_kv_attention(
32
self,
33
- output: torch.Tensor,
34
- query: torch.Tensor,
35
- key: torch.Tensor,
36
- value: torch.Tensor,
33
+ output: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
34
+ query: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
35
+ key: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
36
+ value: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
37
+ prompt_lens: List[int],
38
) -> None:
38
- # FIXME(woosuk): Replace this with a custom op call.
39
- attention_mask = torch.triu(
40
- torch.ones(query.shape[0], key.shape[0]), diagonal=1) * -1e5
41
- attention_mask = attention_mask.to(dtype=query.dtype, device=query.device)
42
- out = self._masked_attention(query, key, value, attention_mask)
43
- output.copy_(out, non_blocking=True)
39
+ # FIXME(woosuk): Replace the following with a custom op.
40
+ start_idx = 0
41
+ for prompt_len in prompt_lens:
42
+ out = output[start_idx:start_idx + prompt_len]
43
+ q = query[start_idx:start_idx + prompt_len]
44
+ k = key[start_idx:start_idx + prompt_len]
45
+ v = value[start_idx:start_idx + prompt_len]
46
+
47
+ attention_mask = torch.triu(
48
+ torch.ones(q.shape[0], k.shape[0]), diagonal=1) * -1e5
49
+ attention_mask = attention_mask.to(dtype=q.dtype, device=q.device)
50
+ attention_out = self._masked_attention(q, k, v, attention_mask)
51
+ out.copy_(attention_out, non_blocking=True)
52
+
53
+ start_idx += prompt_len
54
55
def single_query_cached_kv_attention(
56
self,
47
- output: torch.Tensor,
48
- query: torch.Tensor,
49
- key_cache: torch.Tensor,
50
- value_cache: torch.Tensor,
57
+ output: torch.Tensor, # [num_generation_tokens, num_heads, head_size]
58
+ query: torch.Tensor, # [num_generation_tokens, num_heads, head_size]
59
+ key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
60
+ value_cache: torch.Tensor, # [num_blocks, num_heads, block_size, head_size]
61
input_metadata: InputMetadata,
62
) -> None:
63
num_heads = value_cache.shape[1]
@@ -82,15 +92,18 @@ class OPTCacheFlowAttention(nn.Module):
92
93
def forward(
94
self,
85
- query: torch.Tensor,
86
- key: torch.Tensor,
87
- value: torch.Tensor,
88
- key_cache: torch.Tensor,
89
- value_cache: torch.Tensor,
95
+ query: torch.Tensor, # [num_tokens, num_heads * head_size]
96
+ key: torch.Tensor, # [num_tokens, num_heads * head_size]
97
+ value: torch.Tensor, # [num_tokens, num_heads * head_size]
98
+ key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
99
+ value_cache: torch.Tensor, # [num_blocks, num_heads, block_size, head_size]
100
input_metadata: InputMetadata,
101
cache_event: Optional[torch.cuda.Event],
92
- ) -> torch.Tensor:
93
- # Prune out invalid tokens.
102
+ ) -> torch.Tensor: # [num_tokens, num_heads * head_size]
103
+ # Pre-allocate the output tensor.
104
+ output = torch.empty_like(query)
105
+
106
+ # Prune out paddings if any.
107
query = query[:input_metadata.num_valid_tokens]
108
key = key[:input_metadata.num_valid_tokens]
109
value = value[:input_metadata.num_valid_tokens]
@@ -101,18 +114,11 @@ class OPTCacheFlowAttention(nn.Module):
114
query = query.view(-1, num_heads, head_size)
115
key = key.view(-1, num_heads, head_size)
116
value = value.view(-1, num_heads, head_size)
117
+ output = output.view(-1, num_heads, head_size)
118
119
# Compute the attention op for prompts.
106
- output = torch.empty_like(query)
107
- start_idx = 0
108
- for i in range(input_metadata.num_prompts):
109
- prompt_len = input_metadata.prompt_lens[i]
110
- out = output[start_idx:start_idx + prompt_len]
111
- q = query[start_idx:start_idx + prompt_len]
112
- k = key[start_idx:start_idx + prompt_len]
113
- v = value[start_idx:start_idx + prompt_len]
114
- self.multi_query_kv_attention(out, q, k, v)
115
- start_idx += prompt_len
120
+ self.multi_query_kv_attention(
121
+ output, query, key, value, input_metadata.prompt_lens)
122
123
# Wait until the cache op is done.
124
if cache_event is not None:
@@ -124,6 +130,7 @@ class OPTCacheFlowAttention(nn.Module):
130
131
if input_metadata.num_generation_tokens > 0:
132
# Compute the attention op for generation tokens.
133
+ start_idx = sum(input_metadata.prompt_lens)
134
self.single_query_cached_kv_attention(
135
output[start_idx:],
136
query[start_idx:],
@@ -132,4 +139,5 @@ class OPTCacheFlowAttention(nn.Module):
139
input_metadata)
140
141
# Reshape the output tensor.
142
+ # NOTE(woosuk): The output tensor may include paddings.
143
return output.view(-1, num_heads * head_size)