A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Minor
Woosuk Kwon committed
Feb 14, 2023 at 09:05 UTC
263e91b3864060393b1945c0536005cc0c5afced
1 file changed
+4
-4
cacheflow/master/block_manager.py
+4
-4
@@ -210,8 +210,8 @@ class BlockSpaceManager:
210
}
211
return block_number_mapping
212
213
- def _free_blocks(self, blocks: Iterable[PhysicalTokenBlock]) -> None:
214
- for block in blocks:
213
+ def _free_block_table(self, block_table: BlockTable) -> None:
214
+ for block in block_table:
215
if block.device == Device.GPU:
216
self.gpu_allocator.free(block)
217
else:
@@ -219,10 +219,10 @@ class BlockSpaceManager:
219
220
def free(self, seq: Sequence) -> None:
221
block_table = self.block_tables[seq.seq_id]
222
- self._free_blocks(block_table)
222
+ self._free_block_table(block_table)
223
del self.block_tables[seq.seq_id]
224
225
def reset(self) -> None:
226
for block_table in self.block_tables.values():
227
- self._free_blocks(block_table)
227
+ self._free_block_table(block_table)
228
self.block_tables.clear()