A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix slot mapping
Woosuk Kwon committed
Feb 23, 2023 at 00:10 UTC
4b1ac23f53d0e714a4a48d2c8058438405c0fd07
2 files changed
+16
-6
cacheflow/models/input_metadata.py
+1
-1
@@ -14,7 +14,7 @@ class InputMetadata:
14
block_tables: torch.Tensor,
15
) -> None:
16
self.prompt_lens = prompt_lens
17
- self.prompt_block_table = slot_mapping
17
+ self.slot_mapping = slot_mapping
18
self.context_lens = context_lens
19
self.max_context_len = max_context_len
20
self.block_tables = block_tables
cacheflow/worker/worker.py
+15
-5
@@ -83,12 +83,20 @@ class Worker:
83
generation_seq_ids = sorted(generation_tokens.keys())
84
for seq_id in generation_seq_ids:
85
input_tokens.append(generation_tokens[seq_id])
86
- input_positions.append(context_lens[seq_id] - 1)
87
- generation_block_tables.append(block_tables[seq_id])
86
+ position_id = context_lens[seq_id] - 1
87
+ input_positions.append(position_id)
88
+
89
+ block_table = block_tables[seq_id]
90
+ generation_block_tables.append(block_table)
91
92
max_context_len = max(max_context_len, context_lens[seq_id])
93
max_num_blocks_per_seq = max(
91
- max_num_blocks_per_seq, len(block_tables[seq_id]))
94
+ max_num_blocks_per_seq, len(block_table))
95
+
96
+ block_number = block_table[position_id // self.block_size]
97
+ block_offset = position_id % self.block_size
98
+ slot = block_number * self.block_size + block_offset
99
+ slot_mapping.append(slot)
100
101
# Optimization: Pad the input length to be a multiple of 8.
102
# This is required for utilizing the Tensor Cores in NVIDIA GPUs.
@@ -105,9 +113,11 @@ class Worker:
113
context_lens_tensor = torch.tensor(
114
[context_lens[seq_id] for seq_id in generation_seq_ids],
115
dtype=torch.int, device=self.device)
116
+ padded_block_tables = [
117
+ _pad_to_max(block_table, max_num_blocks_per_seq)
118
+ for block_table in generation_block_tables]
119
block_tables_tensor = torch.tensor(
109
- [_pad_to_max(block_table) for block_table in generation_block_tables],
110
- dtype=int, device=self.device)
120
+ padded_block_tables, dtype=int, device=self.device)
121
122
input_metadata = InputMetadata(
123
prompt_lens=prompt_lens,