@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 263e91b386
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Minor

Woosuk Kwon committed Feb 14, 2023 at 09:05 UTC 263e91b3864060393b1945c0536005cc0c5afced
1 file changed +4 -4
cacheflow/master/block_manager.py
+4 -4
@@ -210,8 +210,8 @@ class BlockSpaceManager:
210 }
211 return block_number_mapping
212
213 - def _free_blocks(self, blocks: Iterable[PhysicalTokenBlock]) -> None:
214 - for block in blocks:
213 + def _free_block_table(self, block_table: BlockTable) -> None:
214 + for block in block_table:
215 if block.device == Device.GPU:
216 self.gpu_allocator.free(block)
217 else:
@@ -219,10 +219,10 @@ class BlockSpaceManager:
219
220 def free(self, seq: Sequence) -> None:
221 block_table = self.block_tables[seq.seq_id]
222 - self._free_blocks(block_table)
222 + self._free_block_table(block_table)
223 del self.block_tables[seq.seq_id]
224
225 def reset(self) -> None:
226 for block_table in self.block_tables.values():
227 - self._free_blocks(block_table)
227 + self._free_block_table(block_table)
228 self.block_tables.clear()