A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix a bug in swap_in and swap_out
Woosuk Kwon committed
Feb 14, 2023 at 02:12 UTC
1f739f9b03663206d5c9c4648d9d5674e695652e
1 file changed
+27
-9
cacheflow/master/block_manager.py
+27
-9
@@ -155,42 +155,60 @@ class BlockSpaceManager:
155
return len(blocks) + num_swapped_seqs <= num_free_blocks
156
157
def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
158
- # src_block_number -> dst_block_number
159
- mapping: Dict[int, int] = {}
158
+ # CPU block -> GPU block.
159
+ mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
160
for seq in seq_group.seqs:
161
if seq.status == SequenceStatus.FINISHED:
162
continue
163
+ new_block_table: BlockTable = []
164
block_table = self.block_tables[seq.seq_id]
165
166
for cpu_block in block_table:
167
if cpu_block in mapping:
168
+ new_block_table.append(mapping[cpu_block])
169
continue
170
gpu_block = self.gpu_allocator.allocate()
169
- mapping[cpu_block.block_number] = gpu_block.block_number
171
+ mapping[cpu_block] = gpu_block
172
# Free the CPU block swapped in to GPU.
173
self.cpu_allocator.free(cpu_block)
172
- return mapping
174
+ self.block_tables[seq.seq_id] = new_block_table
175
+
176
+ block_number_mapping = {
177
+ cpu_block.block_number: gpu_block.block_number
178
+ for cpu_block, gpu_block in mapping.items()
179
+ }
180
+ return block_number_mapping
181
182
def can_swap_out(self, seq_group: SequenceGroup) -> bool:
183
blocks = self._get_physical_blocks(seq_group)
184
return len(blocks) <= self.cpu_allocator.get_num_free_blocks()
185
186
def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
179
- # src_block_number -> dst_block_number
180
- mapping: Dict[int, int] = {}
187
+ # GPU block -> CPU block.
188
+ mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
189
for seq in seq_group.seqs:
190
if seq.status == SequenceStatus.FINISHED:
191
continue
192
+ new_block_table: BlockTable = []
193
block_table = self.block_tables[seq.seq_id]
194
195
for gpu_block in block_table:
187
- if gpu_block.block_number in mapping:
196
+ if gpu_block in mapping:
197
+ new_block_table.append(mapping[gpu_block])
198
continue
199
cpu_block = self.cpu_allocator.allocate()
190
- mapping[gpu_block.block_number] = cpu_block.block_number
200
+ new_block_table.append(cpu_block)
201
+
202
+ mapping[gpu_block] = cpu_block
203
# Free the GPU block swapped out to CPU.
204
self.gpu_allocator.free(gpu_block)
193
- return mapping
205
+ self.block_tables[seq.seq_id] = new_block_table
206
+
207
+ block_number_mapping = {
208
+ gpu_block.block_number: cpu_block.block_number
209
+ for gpu_block, cpu_block in mapping.items()
210
+ }
211
+ return block_number_mapping
212
213
def _free_blocks(self, blocks: Iterable[PhysicalTokenBlock]) -> None:
214
for block in blocks: