A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix attention
Woosuk Kwon committed
Feb 23, 2023 at 23:02 UTC
932844f1cd781aa926439ee2394edfb2f9e696f7
2 files changed
+21
-6
cacheflow/models/attention.py
+10
-6
@@ -53,20 +53,19 @@ class OPTCacheFlowAttention(nn.Module):
53
context_len = int(input_metadata.context_lens[i])
54
55
keys = []
56
+ values = []
57
for j in range(context_len):
57
- block_number = block_table[j // block_size]
58
+ block_number = int(block_table[j // block_size])
59
block_offset = j % block_size
60
+
61
k = key_cache[block_number, :, :, block_offset, :]
62
k = k.reshape(num_heads, head_size)
63
keys.append(k)
62
- keys = torch.stack(keys, dim=0)
64
64
- values = []
65
- for j in range(context_len):
66
- block_number = block_table[j // block_size]
67
- block_offset = j % block_size
65
v = value_cache[block_number, :, block_offset, :]
66
values.append(v)
67
+
68
+ keys = torch.stack(keys, dim=0)
69
values = torch.stack(values, dim=0)
70
71
q = q.unsqueeze(0)
@@ -87,6 +86,11 @@ class OPTCacheFlowAttention(nn.Module):
86
input_metadata: InputMetadata,
87
cache_event: Optional[torch.cuda.Event],
88
) -> torch.Tensor:
89
+ # Prune out invalid tokens.
90
+ query = query[:input_metadata.num_valid_tokens]
91
+ key = key[:input_metadata.num_valid_tokens]
92
+ value = value[:input_metadata.num_valid_tokens]
93
+
94
# Reshape the input tensors.
95
num_heads = value_cache.shape[1]
96
head_size = value_cache.shape[3]
cacheflow/models/input_metadata.py
+11
@@ -11,6 +11,7 @@ class InputMetadata:
11
prompt_lens: List[int],
12
slot_mapping: torch.Tensor,
13
context_lens: torch.Tensor,
14
+ # FIXME: Rename
15
max_context_len: int,
16
block_tables: torch.Tensor,
17
) -> None:
@@ -23,9 +24,19 @@ class InputMetadata:
24
25
self.num_prompts = len(prompt_lens)
26
self.num_generation_tokens = context_lens.shape[0]
27
+ self.num_valid_tokens = len(slot_mapping)
28
if block_tables.numel() > 0:
29
self.max_num_blocks_per_seq = block_tables.shape[1]
30
else:
31
self.max_num_blocks_per_seq = 0
32
assert self.num_generation_tokens == block_tables.shape[0]
33
assert self.num_prompts + self.num_generation_tokens == len(seq_ids)
34
+
35
+ def __repr__(self) -> str:
36
+ return (f'InputMetadata('
37
+ f'seq_ids={self.seq_ids}, '
38
+ f'num_prompts={self.num_prompts}, '
39
+ f'num_generation_tokens={self.num_generation_tokens}, '
40
+ f'num_valid_tokens={self.num_valid_tokens}, '
41
+ f'max_num_blocks_per_seq={self.max_num_blocks_per_seq}, '
42
+ f'max_context_len={self.max_context_len})')