A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix double free
Woosuk Kwon committed
Feb 14, 2023 at 09:14 UTC
c128c2ed0309dd1ff684ac0da2f8fc7ea5994b77
1 file changed
+2
-2
cacheflow/master/block_manager.py
+2
-2
@@ -115,13 +115,13 @@ class BlockSpaceManager:
115
last_block = block_table[-1]
116
assert last_block.device == Device.GPU
117
if last_block.ref_count == 1:
118
- # Append.
118
+ # Not shared with other sequences. Appendable.
119
return None
120
else:
121
# The last block is shared with other sequences.
122
# Copy on Write: Allocate a new block and copy the tokens.
123
new_block = self.gpu_allocator.allocate()
124
- block_table.append(new_block)
124
+ block_table[-1] = new_block
125
self.gpu_allocator.free(last_block)
126
return last_block.block_number, new_block.block_number
127