@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1f739f9b03
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix a bug in swap_in and swap_out

Woosuk Kwon committed Feb 14, 2023 at 02:12 UTC 1f739f9b03663206d5c9c4648d9d5674e695652e
1 file changed +27 -9
cacheflow/master/block_manager.py
+27 -9
@@ -155,42 +155,60 @@ class BlockSpaceManager:
155 return len(blocks) + num_swapped_seqs <= num_free_blocks
156
157 def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
158 - # src_block_number -> dst_block_number
159 - mapping: Dict[int, int] = {}
158 + # CPU block -> GPU block.
159 + mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
160 for seq in seq_group.seqs:
161 if seq.status == SequenceStatus.FINISHED:
162 continue
163 + new_block_table: BlockTable = []
164 block_table = self.block_tables[seq.seq_id]
165
166 for cpu_block in block_table:
167 if cpu_block in mapping:
168 + new_block_table.append(mapping[cpu_block])
169 continue
170 gpu_block = self.gpu_allocator.allocate()
169 - mapping[cpu_block.block_number] = gpu_block.block_number
171 + mapping[cpu_block] = gpu_block
172 # Free the CPU block swapped in to GPU.
173 self.cpu_allocator.free(cpu_block)
172 - return mapping
174 + self.block_tables[seq.seq_id] = new_block_table
175 +
176 + block_number_mapping = {
177 + cpu_block.block_number: gpu_block.block_number
178 + for cpu_block, gpu_block in mapping.items()
179 + }
180 + return block_number_mapping
181
182 def can_swap_out(self, seq_group: SequenceGroup) -> bool:
183 blocks = self._get_physical_blocks(seq_group)
184 return len(blocks) <= self.cpu_allocator.get_num_free_blocks()
185
186 def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
179 - # src_block_number -> dst_block_number
180 - mapping: Dict[int, int] = {}
187 + # GPU block -> CPU block.
188 + mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
189 for seq in seq_group.seqs:
190 if seq.status == SequenceStatus.FINISHED:
191 continue
192 + new_block_table: BlockTable = []
193 block_table = self.block_tables[seq.seq_id]
194
195 for gpu_block in block_table:
187 - if gpu_block.block_number in mapping:
196 + if gpu_block in mapping:
197 + new_block_table.append(mapping[gpu_block])
198 continue
199 cpu_block = self.cpu_allocator.allocate()
190 - mapping[gpu_block.block_number] = cpu_block.block_number
200 + new_block_table.append(cpu_block)
201 +
202 + mapping[gpu_block] = cpu_block
203 # Free the GPU block swapped out to CPU.
204 self.gpu_allocator.free(gpu_block)
193 - return mapping
205 + self.block_tables[seq.seq_id] = new_block_table
206 +
207 + block_number_mapping = {
208 + gpu_block.block_number: cpu_block.block_number
209 + for gpu_block, cpu_block in mapping.items()
210 + }
211 + return block_number_mapping
212
213 def _free_blocks(self, blocks: Iterable[PhysicalTokenBlock]) -> None:
214 for block in blocks: