A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Minor
Woosuk Kwon committed
Feb 9, 2023 at 11:30 UTC
38ed06a846511e5e048da800a28671519457511c
1 file changed
+3
-3
cacheflow/master/block_manager.py
+3
-3
@@ -119,10 +119,10 @@ class BlockSpaceManager:
119
else:
120
# The last block is shared with other sequences.
121
# Copy on Write: Allocate a new block and copy the tokens.
122
- block = self.gpu_allocator.allocate()
123
- block_table.append(block)
122
+ new_block = self.gpu_allocator.allocate()
123
+ block_table.append(new_block)
124
self.gpu_allocator.free(last_block)
125
- return last_block.block_number, block.block_number
125
+ return last_block.block_number, new_block.block_number
126
127
def fork(self, src_seq: Sequence, child_seq: Sequence) -> None:
128
# NOTE: fork does not allocate a new physical block.