@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 932844f1cd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix attention

Woosuk Kwon committed Feb 23, 2023 at 23:02 UTC 932844f1cd781aa926439ee2394edfb2f9e696f7
2 files changed +21 -6
cacheflow/models/attention.py
+10 -6
@@ -53,20 +53,19 @@ class OPTCacheFlowAttention(nn.Module):
53 context_len = int(input_metadata.context_lens[i])
54
55 keys = []
56 + values = []
57 for j in range(context_len):
57 - block_number = block_table[j // block_size]
58 + block_number = int(block_table[j // block_size])
59 block_offset = j % block_size
60 +
61 k = key_cache[block_number, :, :, block_offset, :]
62 k = k.reshape(num_heads, head_size)
63 keys.append(k)
62 - keys = torch.stack(keys, dim=0)
64
64 - values = []
65 - for j in range(context_len):
66 - block_number = block_table[j // block_size]
67 - block_offset = j % block_size
65 v = value_cache[block_number, :, block_offset, :]
66 values.append(v)
67 +
68 + keys = torch.stack(keys, dim=0)
69 values = torch.stack(values, dim=0)
70
71 q = q.unsqueeze(0)
@@ -87,6 +86,11 @@ class OPTCacheFlowAttention(nn.Module):
86 input_metadata: InputMetadata,
87 cache_event: Optional[torch.cuda.Event],
88 ) -> torch.Tensor:
89 + # Prune out invalid tokens.
90 + query = query[:input_metadata.num_valid_tokens]
91 + key = key[:input_metadata.num_valid_tokens]
92 + value = value[:input_metadata.num_valid_tokens]
93 +
94 # Reshape the input tensors.
95 num_heads = value_cache.shape[1]
96 head_size = value_cache.shape[3]
cacheflow/models/input_metadata.py
+11
@@ -11,6 +11,7 @@ class InputMetadata:
11 prompt_lens: List[int],
12 slot_mapping: torch.Tensor,
13 context_lens: torch.Tensor,
14 + # FIXME: Rename
15 max_context_len: int,
16 block_tables: torch.Tensor,
17 ) -> None:
@@ -23,9 +24,19 @@ class InputMetadata:
24
25 self.num_prompts = len(prompt_lens)
26 self.num_generation_tokens = context_lens.shape[0]
27 + self.num_valid_tokens = len(slot_mapping)
28 if block_tables.numel() > 0:
29 self.max_num_blocks_per_seq = block_tables.shape[1]
30 else:
31 self.max_num_blocks_per_seq = 0
32 assert self.num_generation_tokens == block_tables.shape[0]
33 assert self.num_prompts + self.num_generation_tokens == len(seq_ids)
34 +
35 + def __repr__(self) -> str:
36 + return (f'InputMetadata('
37 + f'seq_ids={self.seq_ids}, '
38 + f'num_prompts={self.num_prompts}, '
39 + f'num_generation_tokens={self.num_generation_tokens}, '
40 + f'num_valid_tokens={self.num_valid_tokens}, '
41 + f'max_num_blocks_per_seq={self.max_num_blocks_per_seq}, '
42 + f'max_context_len={self.max_context_len})')